Code-first video & image, rendered natively on WebGPU.
feat(vision-sam): track several SAM 3 objects at once; fix concurrent clip vision
okanaslf1623c09bf76
diff --git a/.changeset/sam3-multi-object.md b/.changeset/sam3-multi-object.mdnew file mode 100644index 00000000..ec524b7e--- /dev/null+++ b/.changeset/sam3-multi-object.md@@ -0,0 +1,7 @@+---+"@framefields/vision-sam": minor+"@framefields/vision": patch+"framefields": minor+---++SAM 3 tracks several objects at once: `sam3.maxObjects` lets a text prompt select up to N instances (tracks 1…N by score) through one encoder pass per frame, and points prompts take `object` to seed or correct one track. `comp.vision` publishes every track in `signals.objects` and `signals.masks`; the clip matte is their union. Fixes concurrent `comp.vision` clips of one source dropping decoded frames (each clip now owns its decoder) and concurrent ORT WebGPU session creation falling back to the CPU (creation is serialized; SAM trackers share the process provider).diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 0bdb05ae..4dad9e80 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -15,9 +15,9 @@ import {VisionRunner,} from "@framefields/vision";import type {- Sam2FrameResult,Sam2Options,Sam2Tracker,+ Sam3ObjectResult,Sam3Options,Sam3Tracker,} from "@framefields/vision-sam";@@ -87,6 +87,20 @@ interface SubjectResult {/** (value, run length) pairs over the source-sized 0–255 mask. */values: Uint8Array;lengths: Uint32Array;+ /**+ * Prompted (SAM) frames only: presence probability per track ID. The runs then hold a+ * label map (track ID per pixel, 0 for none) instead of alpha.+ */+ scores?: number[];+}++/** A subject as written to the subjects cache file. */+interface StoredSubject {+ category: string;+ area: number;+ scores?: number[];+ values: string;+ lengths: string;}interface Transform {@@ -104,6 +118,7 @@ const CACHE_VERSION = 1;type VisionSam = typeof import("@framefields/vision-sam");let visionSam: Promise<VisionSam> | undefined;+let clipVisionIds = 0;/*** SAM lives in `@framefields/vision-sam`, an optional peer under Meta's SAM License, so it@@ -155,10 +170,15 @@ export class ClipVision implements ClipSubjectSource {private sam3Tracker?: Sam3Tracker;private sam3Frame = -1;private sam?: Promise<VisionSam>;+ /** Clips walk their source independently, so each owns a decoder (requests to one supersede). */+ private readonly decoderId = `clip-vision#${++clipVisionIds}#`;private signalBundle?: VisionBundle;private publishedSignals = new Set<number>();private signalMaskFrame = -1;- private signalMask?: InstanceMask;+ /** Masks of `signalMaskFrame` by track ID (0: all tracks merged). */+ private readonly signalMasks = new Map<number, InstanceMask>();+ /** Prompted track count per source frame, including frames where every track is absent. */+ private readonly trackCounts = new Map<number, number>();private cacheLoaded?: Promise<void>;private unsaved = 0;private unsavedSubjects = 0;@@ -232,10 +252,11 @@ export class ClipVision implements ClipSubjectSource {this.signalBundle ??= new VisionBundle({fps: this.compositionFps,totalFrames: Number.MAX_SAFE_INTEGER,- maskSource: (frame) => {- const mask = this.maskAt(frame);- return mask && mask.area > 0 ? [mask] : [];- },+ maskSource: (frame) =>+ this.trackIds(frame).flatMap((id) => {+ const mask = this.maskAt(frame, id);+ return mask && mask.area > 0 ? [mask] : [];+ }),matteSource: (frame) => {const mask = this.maskAt(frame);return mask@@ -251,8 +272,13 @@ export class ClipVision implements ClipSubjectSource {return this.signalBundle;}- /** Free inference sessions. Analysed frames remain readable; later frames replay memory. */+ private decoder(): ReturnType<typeof mediaDecoderCache.getVideo> {+ return mediaDecoderCache.getVideo(this.source, true, this.decoderId);+ }++ /** Free inference sessions and the decoder. Analysed frames remain readable; later frames replay memory. */release(): void {+ mediaDecoderCache.clearNode(this.decoderId);this.sam2Tracker?.release();this.sam2Tracker = undefined;this.sam2Frame = -1;@@ -262,6 +288,7 @@ export class ClipVision implements ClipSubjectSource {this.runner?.close();this.runner = undefined;this.signalMaskFrame = -1;+ this.signalMasks.clear();this.publishedSignals.clear();}@@ -332,7 +359,7 @@ export class ClipVision implements ClipSubjectSource {const height = this.sourceHeight;return {category: s.category,- mask: decodeRuns(s, width * height),+ mask: subjectAlpha(s, width * height),width,height,area: s.area,@@ -431,11 +458,7 @@ export class ClipVision implements ClipSubjectSource {variant: this.options.variant,}),});- const decoder = mediaDecoderCache.getVideo(- this.source,- true,- "clip-vision",- );+ const decoder = this.decoder();for (let c = from; c <= to; c++) {if (this.analysed(c)) continue;const image = await decodedPixels(await decoder.getFrame(c / this.fps));@@ -509,11 +532,7 @@ export class ClipVision implements ClipSubjectSource {this.subjects.set(frame, null);}if (to < first) return;- const decoder = mediaDecoderCache.getVideo(- this.source,- true,- "clip-vision",- );+ const decoder = this.decoder();if (!this.sam2Tracker) {const frameCount = Math.max(1,@@ -547,7 +566,7 @@ export class ClipVision implements ClipSubjectSource {const kp = this.options.pose? await this.matchedPose(image, result.mask): null;- this.storeSam2Frame(frame, result, kp);+ this.storeSamFrame(frame, result.width, result.height, [result], kp);if (this.unsaved >= 24) {await this.saveCache();await this.saveSubjects();@@ -566,11 +585,7 @@ export class ClipVision implements ClipSubjectSource {this.subjects.set(frame, null);}if (to < first) return;- const decoder = mediaDecoderCache.getVideo(- this.source,- true,- "clip-vision",- );+ const decoder = this.decoder();if (!this.sam3Tracker) {const frameCount = Math.max(1,@@ -585,6 +600,7 @@ export class ClipVision implements ClipSubjectSource {baseUrl: this.options.baseUrl,frameCount,textThreshold: settings.textThreshold,+ maxObjects: settings.maxObjects,acceptLicense:this.options.acceptSamLicense ?? this.options.sam3?.acceptLicense,});@@ -599,9 +615,15 @@ export class ClipVision implements ClipSubjectSource {const prompt = settings.prompts.find((item) => item.frame === frame);const result = await this.sam3Tracker.process(image, frame, prompt);const kp = this.options.pose- ? await this.matchedPose(image, result.mask)+ ? await this.matchedPose(image, unionMask(result.objects, image)): null;- this.storeSam2Frame(frame, result, kp);+ this.storeSamFrame(+ frame,+ result.width,+ result.height,+ result.objects,+ kp,+ );if (this.unsaved >= 24) {await this.saveCache();await this.saveSubjects();@@ -611,41 +633,51 @@ export class ClipVision implements ClipSubjectSource {await this.saveSubjects();}- private storeSam2Frame(+ /** Stores a prompted frame: every track's mask as one label map, the union as the subject. */+ private storeSamFrame(frame: number,- result: Sam2FrameResult,+ width: number,+ height: number,+ objects: readonly Pick<Sam3ObjectResult, "mask" | "score">[],kp: Float32Array | null,): void {+ const labels = new Uint8Array(width * height);+ // Ascending presence, so where tracks overlap the most confident one keeps the pixel.+ const order = [...objects.keys()].sort(+ (a, b) => (objects[a]?.score ?? 0) - (objects[b]?.score ?? 0),+ );+ for (const index of order) {+ const mask = objects[index]?.mask;+ if (!mask) continue;+ for (let i = 0; i < mask.length; i++)+ if ((mask[i] ?? 0) >= 128) labels[i] = index + 1;+ }+ const union = Uint8Array.from(labels, (id) => (id ? 255 : 0));const grid = new Uint8Array(this.cellW * this.cellH);downsampleMean(- result.mask,- result.width,- result.height,+ union,+ width,+ height,this.cell,grid,this.cellW,this.cellH,);let area = 0;- for (const alpha of result.mask) area += alpha / 255;+ for (const id of labels) if (id) area++;+ const scores = objects.map((o) => 1 / (1 + Math.exp(-o.score)));this.results.set(frame, {kp,mask: grid,- score: 1 / (1 + Math.exp(-result.score)),+ score: Math.max(0, ...scores),});this.subjects.set(frame,area > 0- ? {- category:- this.options.sam2?.category ??- this.options.sam3?.category ??- "object",- area,- ...encodeRuns(result.mask),- }+ ? { category: this.samCategory(), area, scores, ...encodeRuns(labels) }: null,);+ this.trackCounts.set(frame, objects.length);this.blurred.delete(frame);if (this.options.sam3) this.sam3Frame = frame;else this.sam2Frame = frame;@@ -653,6 +685,12 @@ export class ClipVision implements ClipSubjectSource {this.unsavedSubjects++;}+ private samCategory(): string {+ return (+ this.options.sam2?.category ?? this.options.sam3?.category ?? "object"+ );+ }+private async matchedPose(image: {data: Uint8Array | Uint8ClampedArray;@@ -693,30 +731,44 @@ export class ClipVision implements ClipSubjectSource {);}- private maskAt(frame: number): InstanceMask | undefined {+ /** Track IDs at a composition frame: every prompted track, else the one subject. */+ private trackIds(frame: number): number[] {+ const c = Math.max(0, Math.floor(this.clipFrame(frame)));+ const count =+ this.subjects.get(c)?.scores?.length ?? this.trackCounts.get(c) ?? 1;+ return Array.from({ length: count }, (_, i) => i + 1);+ }++ /** The subject (all tracks merged) or, with `id`, one track's mask, in source pixels. */+ private maskAt(frame: number, id?: number): InstanceMask | undefined {const c = Math.max(0, Math.floor(this.clipFrame(frame)));- if (this.signalMaskFrame === c) return this.signalMask;+ if (this.signalMaskFrame !== c) {+ this.signalMaskFrame = c;+ this.signalMasks.clear();+ }+ const hit = this.signalMasks.get(id ?? 0);+ if (hit) return hit;if (!this.results.has(c) || !this.sourceWidth) return undefined;const stored = this.subjects.get(c);- const data = stored- ? decodeRuns(stored, this.sourceWidth * this.sourceHeight)- : new Uint8Array(this.sourceWidth * this.sourceHeight);- this.signalMaskFrame = c;- this.signalMask = {- category:- stored?.category ??- this.options.sam2?.category ??- this.options.sam3?.category ??- "object",+ const size = this.sourceWidth * this.sourceHeight;+ const data = stored ? subjectAlpha(stored, size, id) : new Uint8Array(size);+ let area = stored?.area ?? 0;+ if (id !== undefined && stored?.scores) {+ area = 0;+ for (const alpha of data) if (alpha) area++;+ }+ const mask: InstanceMask = {+ category: stored?.category ?? this.samCategory(),mask: data,width: this.sourceWidth,height: this.sourceHeight,- area: stored?.area ?? 0,- coverage: (stored?.area ?? 0) / data.length,- detectionIndex: 0,- trackId: 1,+ area,+ coverage: area / size,+ detectionIndex: (id ?? 1) - 1,+ trackId: id ?? 1,};- return this.signalMask;+ this.signalMasks.set(id ?? 0, mask);+ return mask;}private publishSignals(from: number, to: number): void {@@ -729,17 +781,24 @@ export class ClipVision implements ClipSubjectSource {const c = Math.max(0, Math.floor(this.clipFrame(frame)));const mask = this.maskAt(frame);if (!mask) continue;- const previous = bundle- .getObjectResult(frame - 1)- .objects.find((object) => object.trackId === 1);- const object = maskObject(- mask,- frame,- this.compositionFps,- this.results.get(c)?.score ?? 1,- previous,- );- bundle.setObjectResult(frame, [object]);+ const before = bundle.getObjectResult(frame - 1).objects;+ const scores = this.subjects.get(c)?.scores;+ const objects = this.trackIds(frame).flatMap((id) => {+ const track = this.maskAt(frame, id);+ if (!track) return [];+ return [+ maskObject(+ track,+ frame,+ this.compositionFps,+ scores?.[id - 1] ?? this.results.get(c)?.score ?? 1,+ before.find((object) => object.trackId === id),+ ),+ ];+ });+ bundle.setObjectResult(frame, objects);+ const object =+ objects[0] ?? maskObject(mask, frame, this.compositionFps, 0);const kp = this.results.get(c)?.kp;bundle.setPoseResult(frame, {people: kp@@ -875,8 +934,9 @@ export class ClipVision implements ClipSubjectSource {this.cell,o.variant ?? "s",o.confidence ?? null,- ...(o.sam2 ? [[sam?.SAM2_TINY_REVISION, o.sam2]] : []),- ...(o.sam3 ? [[sam?.SAM3_REVISION, o.sam3]] : []),+ // Prompted subjects are track-ID label maps (format 2).+ ...(o.sam2 ? [[sam?.SAM2_TINY_REVISION, o.sam2, 2]] : []),+ ...(o.sam3 ? [[sam?.SAM3_REVISION, o.sam3, 2]] : []),]);const hash = crypto.createHash("sha256")@@ -967,15 +1027,7 @@ export class ClipVision implements ClipSubjectSource {const json = JSON.parse(fs.readFileSync(file, "utf8")) as {width: number;height: number;- frames: Record<- string,- {- category: string;- area: number;- values: string;- lengths: string;- } | null- >;+ frames: Record<string, StoredSubject | null>;};if (this.sourceWidth && json.width !== this.sourceWidth) return;if (!this.sourceWidth) this.setSize(json.width, json.height);@@ -986,6 +1038,7 @@ export class ClipVision implements ClipSubjectSource {s && {category: s.category,area: s.area,+ ...(s.scores && { scores: s.scores }),values: new Uint8Array(Buffer.from(s.values, "base64")),lengths: new Uint32Array(lengths.buffer.slice(@@ -1005,14 +1058,12 @@ export class ClipVision implements ClipSubjectSource {this.unsavedSubjects = 0;const file = await this.subjectsPath();if (!file || !this.sourceWidth) return;- const frames: Record<- string,- { category: string; area: number; values: string; lengths: string } | null- > = {};+ const frames: Record<string, StoredSubject | null> = {};for (const [c, s] of this.subjects) {frames[c] = s && {category: s.category,area: s.area,+ ...(s.scores && { scores: s.scores }),values: Buffer.from(s.values).toString("base64"),lengths: Buffer.from(s.lengths.buffer,@@ -1445,3 +1496,28 @@ export function seedClipVision(});return clip;}++/** A stored subject as 0/255 alpha: label maps merge every track, or keep track `id` only. */+function subjectAlpha(+ stored: SubjectResult,+ size: number,+ id?: number,+): Uint8Array {+ const values = decodeRuns(stored, size);+ if (!stored.scores) return values;+ return values.map((label) =>+ (id === undefined ? label > 0 : label === id) ? 255 : 0,+ );+}++/** Every track's mask merged, for matching one pose to the whole selection. */+function unionMask(+ objects: readonly Pick<Sam3ObjectResult, "mask">[],+ image: { width: number; height: number },+): Uint8Array {+ const union = new Uint8Array(image.width * image.height);+ for (const { mask } of objects)+ for (let i = 0; i < union.length; i++)+ union[i] = Math.max(union[i] ?? 0, mask[i] ?? 0);+ return union;+}diff --git a/packages/framefields/src/sam2-scheduling.test.ts b/packages/framefields/src/sam2-scheduling.test.tsindex 048e13ef..6521fdfa 100644--- a/packages/framefields/src/sam2-scheduling.test.ts+++ b/packages/framefields/src/sam2-scheduling.test.ts@@ -1,7 +1,11 @@import { mkdtemp, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";-import { type Sam2Point, Sam2Tracker } from "@framefields/vision-sam";+import {+ type Sam2Point,+ Sam2Tracker,+ Sam3Tracker,+} from "@framefields/vision-sam";import { mediaDecoderCache } from "@framefields/webgpu-renderers";import { afterEach, describe, expect, it, vi } from "vitest";import { ClipVision } from "./clip-vision.js";@@ -80,4 +84,77 @@ describe("SAM source scheduling and persistence", () => {await rm(dir, { recursive: true, force: true });}});++ it("gives concurrent clips of one source their own decoders", async () => {+ // A decoder drops a pending request when a newer one arrives, so shared+ // decoders made one clip read `null` frames while another advanced.+ vi.spyOn(Sam2Tracker, "create").mockResolvedValue({+ process: async () => ({+ mask: new Uint8Array(32),+ width: 8,+ height: 4,+ score: 1,+ iou: 1,+ }),+ release: () => {},+ } as unknown as Sam2Tracker);+ const getVideo = vi.spyOn(mediaDecoderCache, "getVideo").mockReturnValue({+ durationSec: async () => 1,+ getFrame: async () => ({+ width: 8,+ height: 4,+ buffer: new Uint8Array(128),+ }),+ } as unknown as ReturnType<typeof mediaDecoderCache.getVideo>);+ const clips = [0.25, 0.75].map(+ (x) =>+ new ClipVision("shared.mp4", 24, {+ cacheDir: false,+ sam2: { prompts: [{ frame: 0, points: [{ x, y: 0.5, label: 1 }] }] },+ }),+ );+ await Promise.all(clips.map((clip) => clip.ready(2)));+ const ids = new Set(getVideo.mock.calls.map((call) => call[2]));+ expect(ids.size).toBe(2);+ });++ it("publishes every SAM 3 track under its own ID", async () => {+ const columns = (from: number) =>+ Uint8Array.from({ length: 32 }, (_, i) =>+ i % 8 >= from && i % 8 < from + 2 ? 255 : 0,+ );+ vi.spyOn(Sam3Tracker, "create").mockResolvedValue({+ process: async () => ({+ width: 8,+ height: 4,+ objects: [+ { id: 1, mask: columns(0), score: 3, iou: 1 },+ { id: 2, mask: columns(5), score: 2, iou: 1 },+ ],+ }),+ release: () => {},+ } as unknown as Sam3Tracker);+ vi.spyOn(mediaDecoderCache, "getVideo").mockReturnValue({+ durationSec: async () => 1,+ getFrame: async () => ({+ width: 8,+ height: 4,+ buffer: new Uint8Array(128),+ }),+ } as unknown as ReturnType<typeof mediaDecoderCache.getVideo>);+ const clip = new ClipVision("crowd.mp4", 24, {+ cacheDir: false,+ sam3: { category: "person", maxObjects: 2, text: "student" },+ });+ const signals = clip.signals;+ await clip.ready(0);+ const objects = signals.getObjectResult(0).objects;+ expect(objects.map((o) => [o.trackId, o.boundingBox.originX])).toEqual([+ [1, 0],+ [2, 5],+ ]);+ expect(signals.getMaskResult(0).map((m) => m.trackId)).toEqual([1, 2]);+ expect(signals.segmentation.matte.at(0)?.coverage).toBe(0.5);+ clip.release();+ });});diff --git a/packages/vision-sam/src/sam2/tracker.ts b/packages/vision-sam/src/sam2/tracker.tsindex 074996f2..2ad52e68 100644--- a/packages/vision-sam/src/sam2/tracker.ts+++ b/packages/vision-sam/src/sam2/tracker.ts@@ -7,7 +7,7 @@*/import {- AutoSessionProvider,+ createDefaultSessionProvider,imageToTensor,type SessionProvider,type VisionImageInput,@@ -119,7 +119,7 @@ export class Sam2Tracker {)throw new Error("SAM 2 frameCount must be a positive integer.");const tracker = new Sam2Tracker(constants, options.frameCount);- const provider = options.provider ?? new AutoSessionProvider();+ const provider = options.provider ?? createDefaultSessionProvider();try {for (const graph of GRAPHS) {const session = await provider.createSession(diff --git a/packages/vision-sam/src/sam3/sam3.e2e.test.ts b/packages/vision-sam/src/sam3/sam3.e2e.test.tsindex 89ed2653..5f937e8c 100644--- a/packages/vision-sam/src/sam3/sam3.e2e.test.ts+++ b/packages/vision-sam/src/sam3/sam3.e2e.test.ts@@ -47,9 +47,44 @@ describe.skipIf(!enabled)("SAM 3 / 3.1 on native Dawn WebGPU", () => {frame,frame === 0 ? { points: [{ x: 0.28, y: 0.5, label: 1 }] } : undefined,);- expect(result.score).toBeGreaterThan(0);- expect(result.mask[48 * 128 + 36 + frame * 3]).toBe(255);- expect(result.mask[48 * 128 + 110]).toBe(0);+ const [selected] = result.objects;+ expect(result.objects).toHaveLength(1);+ expect(selected?.score).toBeGreaterThan(0);+ expect(selected?.mask[48 * 128 + 36 + frame * 3]).toBe(255);+ expect(selected?.mask[48 * 128 + 110]).toBe(0);+ }+ } finally {+ tracker.release();+ }+ }, 120_000);++ it("tracks two prompted objects with separate memories", async () => {+ const tracker = await Sam3Tracker.create({+ provider: new NodeWebGPUProvider(),+ frameCount: 8,+ acceptLicense: true,+ });+ try {+ for (let frame = 0; frame < 8; frame++) {+ // Seed object 1 on frame 0 and add object 2 on frame 1.+ const prompt =+ frame === 0+ ? { points: [{ x: 0.28, y: 0.5, label: 1 as const }] }+ : frame === 1+ ? { object: 2, points: [{ x: 0.86, y: 0.5, label: 1 as const }] }+ : undefined;+ const { objects } = await tracker.process(+ picture(frame),+ frame,+ prompt,+ );+ const [moving, still] = objects;+ expect(moving?.mask[48 * 128 + 36 + frame * 3]).toBe(255);+ expect(moving?.mask[48 * 128 + 110]).toBe(0);+ if (frame === 0) continue;+ expect(objects).toHaveLength(2);+ expect(still?.mask[48 * 128 + 110]).toBe(255);+ expect(still?.mask[48 * 128 + 36 + frame * 3]).toBe(0);}} finally {tracker.release();diff --git a/packages/vision-sam/src/sam3/text-selector.ts b/packages/vision-sam/src/sam3/text-selector.tsindex 572e20b3..9a162b6f 100644--- a/packages/vision-sam/src/sam3/text-selector.ts+++ b/packages/vision-sam/src/sam3/text-selector.ts@@ -78,11 +78,17 @@ export class Sam3TextSelector {}}+ /**+ * Masks (288², binary) of the instances matching `text` with sigmoid score of at least+ * `threshold`, best first and at most `max`. Queries that repeat an accepted instance+ * (mask IoU above 0.5) are skipped.+ */async select(pixels: Float32Array,text: string,threshold: number,- ): Promise<Float32Array> {+ max = 1,+ ): Promise<Float32Array[]> {const [visionSession, textSession, decoder] = this.sessions;if (!visionSession || !textSession || !decoder)throw new Error("SAM 3 text selector has been released.");@@ -121,22 +127,30 @@ export class Sam3TextSelector {inputs.push({ name, data: sam3Floats(vision, name), dims: value.dims });}const result = await sam3Run(decoder, inputs),- scores = sam3Floats(result, "pred_logits");- let best = 0;- for (let i = 1; i < scores.length; i++)- if ((scores[i] ?? -Infinity) > (scores[best] ?? -Infinity)) best = i;- const probability = 1 / (1 + Math.exp(-(scores[best] ?? -Infinity)));- if (probability < threshold)- throw new Error(- `SAM 3 found no confident instance for text ${JSON.stringify(text)} (score ${probability.toFixed(3)}, threshold ${threshold}).`,- );- const masks = sam3Floats(result, "pred_masks");- if (masks.length !== scores.length * 288 * 288)+ scores = sam3Floats(result, "pred_logits"),+ masks = sam3Floats(result, "pred_masks");+ if (masks.length !== scores.length * CELLS)throw new Error("Invalid SAM 3 concept mask shape.");- return Float32Array.from(- masks.subarray(best * 288 * 288, (best + 1) * 288 * 288),- (v) => (v > 0 ? 1 : 0),+ const ranked = [...scores.keys()].sort(+ (a, b) => (scores[b] ?? -Infinity) - (scores[a] ?? -Infinity),);+ const best = 1 / (1 + Math.exp(-(scores[ranked[0] ?? 0] ?? -Infinity)));+ if (best < threshold)+ throw new Error(+ `SAM 3 found no confident instance for text ${JSON.stringify(text)} (score ${best.toFixed(3)}, threshold ${threshold}).`,+ );+ const logit = Math.log(threshold / (1 - threshold));+ const selected: Float32Array[] = [];+ for (const query of ranked) {+ if (selected.length >= max || (scores[query] ?? -Infinity) < logit) break;+ const candidate = Float32Array.from(+ masks.subarray(query * CELLS, (query + 1) * CELLS),+ (v) => (v > 0 ? 1 : 0),+ );+ if (selected.every((kept) => maskIou(kept, candidate) <= 0.5))+ selected.push(candidate);+ }+ return selected;}release(): void {@@ -144,3 +158,17 @@ export class Sam3TextSelector {this.sessions.length = 0;}}++const CELLS = 288 * 288;++function maskIou(a: Float32Array, b: Float32Array): number {+ let both = 0,+ either = 0;+ for (let i = 0; i < a.length; i++) {+ const x = a[i] ?? 0,+ y = b[i] ?? 0;+ both += x * y;+ either += Math.max(x, y);+ }+ return either ? both / either : 0;+}diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.tsindex d9419da1..8deb10ce 100644--- a/packages/vision-sam/src/sam3/tracker.ts+++ b/packages/vision-sam/src/sam3/tracker.ts@@ -7,7 +7,7 @@*/import {- AutoSessionProvider,+ createDefaultSessionProvider,imageToTensor,type SessionProvider,type VisionImageInput,@@ -20,11 +20,16 @@ import {import { assertSamLicenseAccepted } from "../license.js";import { ensureSamModelsRegistered } from "../models/registry.js";import { sam2Tokens } from "../sam2/memory.js";-import type { Sam2FrameResult } from "../sam2/types.js";+import type { Sam2Point } from "../sam2/types.js";import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";import { Sam3MemoryBank, sam3Position } from "./memory.js";import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";-import { type Sam3Prompt, validateSam3Prompts } from "./types.js";+import {+ type Sam3FrameResult,+ type Sam3ObjectResult,+ type Sam3Prompt,+ validateSam3Prompts,+} from "./types.js";/** All graph and tokenizer revisions participate in ClipVision's persistent cache identity. */export const SAM3_REVISION =@@ -35,6 +40,8 @@ export interface Sam3TrackerOptions extends VisionModelStoreOptions {readonly store?: VisionModelStore;readonly frameCount?: number;readonly textThreshold?: number;+ /** Most instances a text prompt selects. Default 1. */+ readonly maxObjects?: number;/** Explicitly agree to Meta's SAM License terms to download or run SAM models. */readonly acceptLicense?: boolean;}@@ -48,6 +55,12 @@ type Graph =| "memory_encoder"| "memory_attention";+/** How an object is (re)seeded on a prompted frame. */+interface Seed {+ readonly points?: readonly Sam2Point[];+ readonly dense?: Float32Array;+}+function tensor(name: string,data: VisionTensorInput["data"],@@ -56,11 +69,15 @@ function tensor(return { name, data, dims };}-/** One explicitly selected SAM 3 instance, with full neural memory propagation on the provider. */+/**+ * Explicitly selected SAM 3 instances with full neural memory propagation on the provider.+ * The image encoder runs once per frame; each object keeps its own memory bank.+ */export class Sam3Tracker {private readonly sessions = new Map<Graph, VisionSession>();private readonly position = sam3Position();- private bank?: Sam3MemoryBank;+ private readonly banks: Sam3MemoryBank[] = [];+ private temporal?: Float32Array;private noMemory?: Float32Array;private selector?: Sam3TextSelector;private lastFrame = -1;@@ -87,6 +104,11 @@ export class Sam3Tracker {options.textThreshold >= 1))throw new Error("SAM 3 textThreshold must be between zero and one.");+ if (+ options.maxObjects !== undefined &&+ (!Number.isSafeInteger(options.maxObjects) || options.maxObjects < 1)+ )+ throw new Error("SAM 3 maxObjects must be a positive integer.");const store =options.store ??@@ -94,7 +116,7 @@ export class Sam3Tracker {...options,acceptLicense: options.acceptLicense ?? true,});- const provider = options.provider ?? new AutoSessionProvider();+ const provider = options.provider ?? createDefaultSessionProvider();const tracker = new Sam3Tracker(store, provider, options);try {tracker.sessions.set(@@ -115,7 +137,7 @@ export class Sam3Tracker {await store.ensure("sam3-support"),await store.ensure("sam3-support-data"),);- tracker.bank = new Sam3MemoryBank(support.temporal);+ tracker.temporal = support.temporal;tracker.noMemory = support.noMemory;tracker.sessions.set("dense_prompt",@@ -160,7 +182,7 @@ export class Sam3Tracker {image: VisionImageInput,frame: number,prompt?: Omit<Sam3Prompt, "frame">,- ): Promise<Sam2FrameResult> {+ ): Promise<Sam3FrameResult> {if (this.released) throw new Error("SAM 3 tracker has been released.");if (!Number.isSafeInteger(frame) ||@@ -184,12 +206,8 @@ export class Sam3Tracker {const vision = await this.run("vision_encoder", [tensor("pixel_values", pixels, [1, 3, 1008, 1008]),]);- const raw = sam3Floats(vision, "image_embeddings.2"),- features = prompt- ? this.initialFeatures(raw)- : await this.condition(raw, frame);- const points = prompt?.points ?? [{ x: 0, y: 0, label: -1 }];- const inputs = [+ const raw = sam3Floats(vision, "image_embeddings.2");+ const shared = [tensor("feats0",sam3Floats(vision, "image_embeddings.0"),@@ -200,61 +218,102 @@ export class Sam3Tracker {sam3Floats(vision, "image_embeddings.1"),[1, 64, 144, 144],),- tensor("feats2", features, [1, 256, 72, 72]),- tensor(- "input_points",- Float32Array.from(points.flatMap((p) => [p.x * 1008, p.y * 1008])),- [1, 1, points.length, 2],- ),- tensor("input_labels", Int32Array.from(points.map((p) => p.label)), [- 1,- 1,- points.length,- ]),];+ const seeds = await this.seeds(pixels, prompt);+ const objects: Sam3ObjectResult[] = [];+ for (const [index, bank] of this.banks.entries()) {+ const seed = seeds.get(index);+ const features = seed+ ? this.initialFeatures(raw)+ : await this.condition(raw, frame, bank);+ const decoded = await this.decode(shared, features, seed);+ await this.remember(bank, raw, decoded, frame, seed !== undefined);+ const score = sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity;+ objects.push({+ id: index + 1,+ mask: sam3Mask(+ sam3Floats(decoded, "high_res_masks"),+ image.width,+ image.height,+ score,+ ),+ score,+ iou: Math.max(...sam3Floats(decoded, "iou_scores")),+ });+ }+ this.lastFrame = frame;+ return { width: image.width, height: image.height, objects };+ }+ /** Objects a prompt (re)seeds, by index. Text re-selects every object; points address one. */+ private async seeds(+ pixels: Float32Array,+ prompt?: Omit<Sam3Prompt, "frame">,+ ): Promise<Map<number, Seed>> {+ const seeds = new Map<number, Seed>();+ if (prompt?.points) {+ const index = (prompt.object ?? 1) - 1;+ if (index > this.banks.length)+ throw new Error(+ `SAM 3 prompt object ${index + 1} skips track IDs; the next new ID is ${this.banks.length + 1}.`,+ );+ if (index === this.banks.length) this.banks.push(this.newBank());+ seeds.set(index, { points: prompt.points });+ }if (prompt?.text !== undefined) {this.selector ??= await Sam3TextSelector.create(this.store,this.provider,);if (!this.options.store) this.store.clearMemoryCache();- const mask = await this.selector.select(+ const masks = await this.selector.select(pixels,prompt.text,this.options.textThreshold ?? 0.5,+ this.options.maxObjects ?? 1,);- const dense = await this.run("dense_prompt", [- tensor("mask", mask, [1, 1, 288, 288]),- ]);- inputs.push(- tensor(- "dense_prompt",- sam3Floats(dense, "dense_prompt"),- [1, 256, 72, 72],- ),- );+ this.banks.length = 0;+ for (const [index, mask] of masks.entries()) {+ this.banks.push(this.newBank());+ const dense = await this.run("dense_prompt", [+ tensor("mask", mask, [1, 1, 288, 288]),+ ]);+ seeds.set(index, { dense: sam3Floats(dense, "dense_prompt") });+ }}+ return seeds;+ }- const decoded = await this.run(- prompt?.text !== undefined ? "dense_decoder" : "mask_decoder",- inputs,- );- await this.remember(raw, decoded, frame, prompt !== undefined);- this.lastFrame = frame;- const score = sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity;- return {- mask: sam3Mask(- sam3Floats(decoded, "high_res_masks"),- image.width,- image.height,- score,+ private newBank(): Sam3MemoryBank {+ if (!this.temporal) throw new Error("SAM 3 tracker has been released.");+ return new Sam3MemoryBank(this.temporal);+ }++ private decode(+ shared: readonly VisionTensorInput[],+ features: Float32Array,+ seed?: Seed,+ ): Promise<VisionTensorOutputs> {+ const points = seed?.points ?? [{ x: 0, y: 0, label: -1 }];+ const inputs = [+ ...shared,+ tensor("feats2", features, [1, 256, 72, 72]),+ tensor(+ "input_points",+ Float32Array.from(points.flatMap((p) => [p.x * 1008, p.y * 1008])),+ [1, 1, points.length, 2],),- width: image.width,- height: image.height,- score,- iou: Math.max(...sam3Floats(decoded, "iou_scores")),- };+ tensor("input_labels", Int32Array.from(points.map((p) => p.label)), [+ 1,+ 1,+ points.length,+ ]),+ ];+ if (!seed?.dense) return this.run("mask_decoder", inputs);+ return this.run("dense_decoder", [+ ...inputs,+ tensor("dense_prompt", seed.dense, [1, 256, 72, 72]),+ ]);}private initialFeatures(raw: Float32Array): Float32Array {@@ -268,9 +327,9 @@ export class Sam3Tracker {private async condition(raw: Float32Array,frame: number,+ memory: Sam3MemoryBank,): Promise<Float32Array> {- const bank = this.bank?.assemble(frame, this.options.frameCount ?? 16);- if (!bank) throw new Error("Missing SAM 3 memory bank.");+ const bank = memory.assemble(frame, this.options.frameCount ?? 16);const positions = await this.run("pointer_position", [tensor("encoding", bank.encoding, [16, 256]),]);@@ -311,6 +370,7 @@ export class Sam3Tracker {}private async remember(+ bank: Sam3MemoryBank,raw: Float32Array,decoded: VisionTensorOutputs,frame: number,@@ -330,7 +390,7 @@ export class Sam3Tracker {),tensor("is_mask_from_pts", new Uint8Array([prompted ? 1 : 0]), [1]),]);- this.bank?.remember(+ bank.remember({frame,tokens: sam3Floats(memory, "memory_tokens").slice(),@@ -345,7 +405,8 @@ export class Sam3Tracker {for (const session of this.sessions.values()) session.release();this.sessions.clear();this.selector?.release();- this.bank = undefined;+ this.banks.length = 0;+ this.temporal = undefined;this.noMemory = undefined;this.released = true;}diff --git a/packages/vision-sam/src/sam3/types.ts b/packages/vision-sam/src/sam3/types.tsindex 85dde21f..2f214d1b 100644--- a/packages/vision-sam/src/sam3/types.ts+++ b/packages/vision-sam/src/sam3/types.ts@@ -15,8 +15,13 @@ export interface Sam3Prompt {readonly frame: number;/** Normalized source coordinates; labels 2/3 define a box. */readonly points?: readonly Sam2Point[];- /** Select the highest-scoring semantic instance, then propagate its mask. */+ /** Select matching instances (up to `maxObjects`, best first), then propagate their masks. */readonly text?: string;+ /**+ * Track ID (1-based) a points prompt seeds or corrects; defaults to 1. The next unused+ * ID adds an object. Text prompts re-select every object, so they take no ID.+ */+ readonly object?: number;}export interface Sam3Options extends SamLicenseOptions {@@ -26,8 +31,10 @@ export interface Sam3Options extends SamLicenseOptions {readonly text?: string;/** Signal category metadata; defaults to "object". */readonly category?: string;- /** Minimum sigmoid score for the initial text-selected instance. Default 0.5. */+ /** Minimum sigmoid score for a text-selected instance. Default 0.5. */readonly textThreshold?: number;+ /** Most instances a text prompt selects, as track IDs 1…N by score. Default 1. */+ readonly maxObjects?: number;}export function validateSam3Prompts(options: Sam3Options): void {@@ -48,8 +55,38 @@ export function validateSam3Prompts(options: Sam3Options): void {validateSam2Prompts([{ frame: prompt.frame, points: prompt.points }]);if (prompt.text !== undefined && !prompt.text.trim())throw new Error("SAM 3 text prompts must not be empty.");+ if (+ prompt.object !== undefined &&+ (prompt.text !== undefined ||+ !Number.isSafeInteger(prompt.object) ||+ prompt.object < 1)+ )+ throw new Error(+ "SAM 3 prompt objects are positive track IDs on points prompts.",+ );}+ const maxObjects = options.maxObjects ?? 1;+ if (!Number.isSafeInteger(maxObjects) || maxObjects < 1)+ throw new Error("SAM 3 maxObjects must be a positive integer.");const threshold = options.textThreshold ?? 0.5;if (!Number.isFinite(threshold) || threshold <= 0 || threshold >= 1)throw new Error("SAM 3 textThreshold must be between zero and one.");}++/** One tracked SAM 3 object at a frame; an empty mask means it is absent there. */+export interface Sam3ObjectResult {+ /** Track ID, 1-based and stable until a text prompt re-selects every object. */+ readonly id: number;+ /** Source-sized binary mask (0 or 255). */+ readonly mask: Uint8Array;+ /** Object-presence logit; positive means present. */+ readonly score: number;+ readonly iou: number;+}++export interface Sam3FrameResult {+ readonly width: number;+ readonly height: number;+ /** Every tracked object, ordered by track ID. */+ readonly objects: readonly Sam3ObjectResult[];+}diff --git a/packages/vision/src/runtime/webgpu-provider.test.ts b/packages/vision/src/runtime/webgpu-provider.test.tsindex fa5da870..a4233e41 100644--- a/packages/vision/src/runtime/webgpu-provider.test.ts+++ b/packages/vision/src/runtime/webgpu-provider.test.ts@@ -58,6 +58,40 @@ function fakeModule(opts: { failWebGPU?: boolean } = {}) {}describe("WebGPUProvider", () => {+ it("makes one ORT call at a time per runtime, as its WebGPU build requires", async () => {+ const { module } = fakeModule();+ // The real runtime throws on a concurrent create and corrupts its heap on a concurrent run.+ let busy = false;+ const exclusive = async <T>(call: () => Promise<T>): Promise<T> => {+ if (busy) throw new Error("concurrent ORT call");+ busy = true;+ await new Promise((resolve) => setTimeout(resolve, 5));+ busy = false;+ return call();+ };+ const create = module.InferenceSession.create;+ module.InferenceSession.create = (bytes, options) =>+ exclusive(async () => {+ const session = await create(bytes, options);+ const run = session.run.bind(session);+ return {+ ...session,+ inputNames: ["input"],+ run: (feeds) => exclusive(() => run(feeds)),+ };+ });+ const providers = [1, 2, 3].map(+ () => new WebGPUProvider({ loader: async () => module }),+ );+ const sessions = await Promise.all(+ providers.map((p) => p.createSession(new Uint8Array([1]))),+ );+ const input = { name: "input", data: new Float32Array([1]), dims: [1] };+ await expect(+ Promise.all(sessions.map((session) => session.run(input))),+ ).resolves.toHaveLength(3);+ });+it("passes multiple named float and integer tensors without renaming", async () => {const { module } = fakeModule();let feeds: Record<string, unknown> = {};diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.tsindex f9b7e753..85a7e138 100644--- a/packages/vision/src/runtime/webgpu-provider.ts+++ b/packages/vision/src/runtime/webgpu-provider.ts@@ -115,17 +115,15 @@ export class WebGPUProvider implements SessionProvider {return this._ortPromise;}- public async createSession(+ private async createOrtSession(+ ort: OrtWebModule,modelBytes: Uint8Array,options?: VisionSessionOptions,- ): Promise<VisionSession> {- const ort = await this.ort();-- let session: OrtWebSession;+ ): Promise<OrtWebSession> {try {const device = this.device;const layout = needsNchwLayout(modelBytes) ? "NCHW" : "NHWC";- session = await ort.InferenceSession.create(modelBytes, {+ return await ort.InferenceSession.create(modelBytes, {executionProviders: this._providers.map((name) =>withProviderOptions(name, device, layout),),@@ -139,7 +137,7 @@ export class WebGPUProvider implements SessionProvider {});} catch (err) {if (!this._wasmFallback || !this._providers.includes("webgpu")) throw err;- session = await ort.InferenceSession.create(modelBytes, {+ return await ort.InferenceSession.create(modelBytes, {executionProviders: ["wasm"],graphOptimizationLevel: "all",externalData: options?.externalData?.map(({ path, data }) => ({@@ -150,6 +148,16 @@ export class WebGPUProvider implements SessionProvider {logSeverityLevel: 3,});}+ }++ public async createSession(+ modelBytes: Uint8Array,+ options?: VisionSessionOptions,+ ): Promise<VisionSession> {+ const ort = await this.ort();+ const session = await exclusive(ort, () =>+ this.createOrtSession(ort, modelBytes, options),+ );const runMany = async (inputs: readonly VisionTensorInput[],@@ -181,7 +189,7 @@ export class WebGPUProvider implements SessionProvider {[...input.dims],);}- const results = await session.run(feeds);+ const results = await exclusive(ort, () => session.run(feeds));const outputs: Record<string, VisionTensorOutputs[string]> = {};for (const [name, tensor] of Object.entries(results)) {outputs[name] = {@@ -195,7 +203,7 @@ export class WebGPUProvider implements SessionProvider {run: (input) => runMany([input]),runMany,release: () => {- void session.release();+ void exclusive(ort, () => session.release());},};}@@ -230,3 +238,19 @@ function withProviderOptions(if (name !== "webgpu") return name;return device ? { name, preferredLayout, device } : { name, preferredLayout };}++/**+ * ORT Web's WebGPU build takes one call at a time per runtime: a concurrent session create+ * throws, and a run entered while another awaits the GPU corrupts its wasm heap. Every+ * create, run and release on one module therefore queues here.+ */+const queues = new WeakMap<OrtWebModule, Promise<unknown>>();++function exclusive<T>(ort: OrtWebModule, call: () => Promise<T>): Promise<T> {+ const result = (queues.get(ort) ?? Promise.resolve()).then(call, call);+ queues.set(+ ort,+ result.catch(() => undefined),+ );+ return result;+}diff --git a/packages/vision/src/signals/mask-object.ts b/packages/vision/src/signals/mask-object.tsindex dafaae1d..6de45d12 100644--- a/packages/vision/src/signals/mask-object.ts+++ b/packages/vision/src/signals/mask-object.ts@@ -1,7 +1,7 @@import type { TrackedObject } from "@framefields/core";import type { InstanceMask } from "../types.js";-/** Tracked-object geometry of one selected mask (bounds, center, velocity), with no detector involved. */+/** Tracked-object geometry of one selected mask (bounds, center, velocity), with no detector involved; the track ID is the mask's (default 1). */export function maskObject(mask: InstanceMask,frame: number,@@ -33,7 +33,7 @@ export function maskObject(const vy =active && previous?.active ? (centerY - previous.centerY) * fps : 0;return {- trackId: 1,+ trackId: mask.trackId ?? 1,category: mask.category,score,boundingBox: {diff --git a/plugins/framefields/skills/framefields-effects/SKILL.md b/plugins/framefields/skills/framefields-effects/SKILL.mdindex 602fea81..16ebd96f 100644--- a/plugins/framefields/skills/framefields-effects/SKILL.md+++ b/plugins/framefields/skills/framefields-effects/SKILL.md@@ -260,6 +260,13 @@ const trackedBox = comp.vision(subject, {},});+// Option C: several instances at once — one SAM 3 pipeline, tracks 1…N by score+const crowd = comp.vision("assets/source.mp4", {+ acceptSamLicense: true,+ sam3: { text: "student in blazer", category: "person", maxObjects: 6 },+});+crowd.signals.objects.get(2); // second-best match; points prompts can add or correct one: { frame, object: 3, points }+const signals = tracked.signals; // stable VisionBundle; request before renderingconst target = signals.objects.get(1);// target.bounds, anchors, confidence, active, kinematics and pose use the existing signals API.@@ -269,7 +276,7 @@ const mask = signals.segmentation.matte.at(0); // source-sized selected-object aCoordinates are normalized to the original source image, before layer placement. Labels are foreground `1`, background `0`, and box corners `2`/`3`. Prompt frames are nonnegative increasing integers on the source analysis clock (`options.fps`, default composition FPS), before trimming/time remapping. SAM propagates forward from the earliest prompt; frames before it are empty. Corrections reset the conditioning memory. Reset/re-prompt across cuts or when identity is lost; appearance tracking cannot guarantee re-identification after occlusion. `sam2` and `sam3` are mutually exclusive on a single clip.-SAM masks do not merge other detected people. Default smoothing is zero and a missing object stays empty. Pose is disabled unless explicitly requested; SAM itself provides no skeleton. `tracked.person` continues to expose silhouette sampling and optional matched pose; `.place(...)` maps those into a layer's layout. The `tracked.signals` bundle uses source geometry on the composition clock; object and mask signals refer to track ID `1`. Full masks are reconstructed lazily from compressed history, avoiding an uncompressed mask per frame.+Text prompts are concepts: describe what sets the target apart ("man with shaved head", not "man in white shirt" in a crowd of white shirts). With `maxObjects`, one encoder pass per frame serves every track; each track keeps its own memory, overlaps go to the more confident track, and the clip's matte/subject is all tracks merged. A later text prompt re-selects every track; a points prompt with `object: k` seeds or corrects only track k (k = next unused ID adds one). SAM masks do not merge other detected people. Default smoothing is zero and a missing object stays empty. Pose is disabled unless explicitly requested; SAM itself provides no skeleton. `tracked.person` continues to expose silhouette sampling and optional matched pose; `.place(...)` maps those into a layer's layout. The `tracked.signals` bundle uses source geometry on the composition clock; object and mask signals refer to track ID `1`. Full masks are reconstructed lazily from compressed history, avoiding an uncompressed mask per frame.Frame requests analyze intervening source frames in order, so a seek forward can take time; a backward seek, hold, reverse, and time-remapped playback reuse source-frame results. When analysis FPS is lower than playback FPS, intermediate frames reuse the preceding analysis mask. Source masks are persisted separately from low-resolution silhouette/pose data; cache identity includes all prompts and the model revision. `pnpm --filter @framefields/vision-sam test:sam2` and `test:sam3` run real-model tests on native Dawn (requires GPU access; downloads are cached). A matte/mask/crop layer of the same clip reuses the selected masks through the existing clip-matte link. Use the default matte settings and matching source aspect ratio for that link; crop after source-space matting. An explicit selection fails clearly instead of falling back to another detected person when the mask cannot align. `tracked.release()` frees SAM sessions, keeps results readable, and replays the source memory if further uncached frames are requested.