Skip to content

Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.

22.1 KB
Raw Blame
import { ApplicationError } from '../../../shared/src/utils/errorSharedUtil';
import { sha256 } from '../../../shared/src/utils/cryptoSharedUtil';
import { RETENTION_MS } from '../../../shared/src/utils/layoutStorageUtil';
import type { ApiPrincipal } from '../models/credentialSharedmodel';
import type { Env } from '../models/applicationSharedmodel';
import type { JsonObject } from '../../../shared/src/models/httpSharedmodel';
import type { ModelVersion } from '../models/modelInferencemodel';
import type {
  WorkshopState,
  WorkshopRuntime,
  WorkshopSnapshot,
  WorkshopModel,
  WorkshopSeed,
  WorkshopInput,
  WorkshopVersionManifest,
  WorkshopItem,
  WorkshopRunOptions,
} from '../models/workshopInferencemodel';
import * as ops from '../db/WorkshopInferenceOps';
import * as models from '../db/ModelInferenceOps';
import { canonicalModel } from '../utils/modelInferenceUtil';
import { cameraIds } from './cameraStorageService';
import { publishRun } from './workshopLiveInferenceService';
import {
  validateDocument,
  executionGraph,
  samplingWindow,
  invalid,
  noMedia,
  identifier,
  hash,
  record,
  usesTracking,
} from './workshopValidationInferenceService';
const now = () => new Date().toISOString();
// What each processor engine can run. v3 tracks objects across a recording: frames kept by change, motion rules.
const engines: Record<string, string[]> = {
  'workshop-v2': ['test-runs', 'live-progress'],
  'workshop-v3': ['test-runs', 'live-progress', 'tracking'],
};
const engineOf = (runtime: WorkshopRuntime) => String(JSON.parse(runtime.manifest_json).engine);
export async function loadWorkshop(env: Env): Promise<WorkshopState> {
  const stored = await ops.state(env);
  return stored
    ? { revision: stored.revision, document: JSON.parse(stored.document_json) }
    : {
        revision: 0,
        document: {
          version: 3,
          workflows: [],
          buckets: [
            {
              id: 'camera-archive',
              name: 'Camera archive',
              kind: 'raw',
              description: 'Original video from your cameras.',
              createdAt: null,
            },
          ],
        },
      };
}
export async function saveWorkshop(env: Env, body: JsonObject) {
  if (!Number.isSafeInteger(body.revision) || Number(body.revision) < 0) invalid('A document revision is required.');
  const document = validateDocument(body.document),
    previous = await loadWorkshop(env);
  if (previous.revision !== body.revision)
    throw new ApplicationError(
      'conflict',
      'Workshop changed in another window. Reload before saving; your browser copy is preserved.',
    );
  if (previous.document.buckets.some(b => !document.buckets.some(next => next.id === b.id && next.kind === b.kind)))
    invalid('Existing buckets cannot be removed or change kind.');
  if (previous.document.workflows.some(w => !document.workflows.some(next => next.id === w.id)))
    invalid('Use the workflow deletion action to remove a workflow and cancel its unfinished runs.');
  const serialized = JSON.stringify(document);
  if (new TextEncoder().encode(serialized).length > 750000) invalid('Workshop is too large.');
  if (!(await ops.saveState(env, Number(body.revision), serialized, now())))
    throw new ApplicationError('conflict', 'Workshop changed in another window. Reload before saving.');
  return { revision: Number(body.revision) + 1, document };
}
export async function runtimeStatus(env: Env) {
  return {
    runtimes: (await ops.runtimes(env)).map(runtime => {
      const { manifest_json, ...r } = runtime;
      return { ...r, capabilities: engines[engineOf(runtime)] ?? [] };
    }),
    enabled: !!env.WORKSHOP,
  };
}
export async function registerRuntime(env: Env, body: JsonObject) {
  // Any registered device ID will do; device IDs may contain dots, which workflow identifiers may not.
  if (!hash(body.code_sha256) || typeof body.device_id !== 'string' || !(await models.findDevice(env, body.device_id)))
    invalid('Register a device and provide the processor code SHA-256.');
  const manifest = {
    schema_version: 1,
    engine: 'workshop-v3',
    device_id: body.device_id,
    code_sha256: body.code_sha256,
  };
  const serialized = canonicalModel(manifest),
    id = `workshop-${(await sha256(serialized)).slice(0, 32)}`;
  const runtime: WorkshopRuntime = {
    id,
    device_id: String(body.device_id),
    code_sha256: String(body.code_sha256),
    manifest_json: serialized,
    created_at: now(),
    last_seen_at: null,
  };
  await ops.registerRuntime(env, runtime);
  return (await ops.runtime(env, id))!;
}
export async function createRun(env: Env, body: JsonObject, options: WorkshopRunOptions = {}) {
  if (!env.WORKSHOP) throw new ApplicationError('unavailable', 'Workshop processing storage is not configured.');
  if (
    !identifier(body.id) ||
    !identifier(body.workflow_id) ||
    !identifier(body.runtime_id) ||
    !Number.isSafeInteger(body.revision)
  )
    invalid('Choose a saved workflow and registered runtime.');
  const mode = body.mode ?? 'run';
  if (mode !== 'test' && mode !== 'run') invalid('Choose test or run mode.');
  if (
    mode === 'test' &&
    (!Number.isInteger(body.image_limit) || Number(body.image_limit) < 1 || Number(body.image_limit) > 100)
  )
    invalid('A test processes between 1 and 100 images.');
  if (mode === 'run' && body.image_limit !== undefined) invalid('Image limits belong to test runs.');
  const limit = mode === 'test' ? Number(body.image_limit) : 2000;
  const previewExpiry = new Date(Date.now() + 3600000).toISOString();
  const requestHash = await sha256(canonicalModel(body)),
    existing = await ops.run(env, String(body.id));
  if (existing) {
    if (existing.request_hash !== requestHash)
      throw new ApplicationError('conflict', 'This run ID has already been used.');
    return existing;
  }
  const state = await loadWorkshop(env);
  if (state.revision !== body.revision)
    throw new ApplicationError('conflict', 'Save and reload the latest workflow before running.');
  const workflow = state.document.workflows.find(w => w.id === body.workflow_id);
  if (!workflow) invalid('Workflow not found.');
  const { input, outputs, footage } = executionGraph(workflow);
  const runtime = await ops.runtime(env, String(body.runtime_id));
  if (!runtime) invalid('Register this processor runtime first.');
  const capabilities = engines[engineOf(runtime)] ?? [];
  if (mode === 'test' && !capabilities.includes('test-runs'))
    invalid('Update the Workshop processor before submitting tests.');
  if (usesTracking(workflow) && !capabilities.includes('tracking'))
    invalid('Update the Workshop processor: this workflow keeps frames by change or routes on motion.');
  const selected: WorkshopModel[] = [];
  for (const node of workflow.nodes.filter(n => n.data.kind === 'model')) {
    const [family, version, ...rest] = String(node.data.modelKey).split('/');
    const row = !rest.length && family && version ? await models.findModel(env, family, version) : null;
    if (!row) invalid('A selected model version is unavailable.');
    const model = JSON.parse(row.manifest_json) as ModelVersion;
    const issue = workshopModelIssue(model);
    if (issue) invalid(issue);
    const artifact = model.artifacts.find(a => a.filename.endsWith('.pt'));
    if (!artifact || !(await models.findArtifact(env, artifact.sha256)))
      invalid(`The weights for ${model.name} are unavailable.`);
    const size = Number(
      model.input.width ??
        model.input.imgsz ??
        (Array.isArray(model.input.training_size) ? model.input.training_size[0] : model.input.training_size) ??
        640,
    );
    const imgsz = Number.isInteger(size) && size >= 32 && size <= 2048 ? Math.ceil(size / 32) * 32 : 640;
    selected.push({
      node_id: node.id,
      model,
      artifact_sha256: artifact.sha256,
      imgsz,
      confidence: Number(node.data.score),
    });
  }
  if (!selected.length || selected.length > 4) invalid('Choose between one and four supported models.');
  const window = footage ? samplingWindow(footage.data, options.now) : null,
    interval = footage ? Number(footage.data.interval) : 1;
  const snapshot: WorkshopSnapshot = {
    schema_version: 1,
    mode,
    ...(mode === 'test' ? { image_limit: limit, preview_expires_at: previewExpiry } : {}),
    workflow,
    models: selected,
    runtime,
    input_bucket: String(input.data.bucketId),
    output_buckets: outputs.map(n => String(n.data.bucketId)),
    interval,
    ...(footage?.data.pick === 'change' ? { pick: 'change' as const } : {}),
    from: window?.from ?? null,
    until: window?.until ?? null,
  };
  const seeds: WorkshopSeed[] = [];
  const created = now();
  let estimated = 0;
  if (window) {
    const camera = String(input.data.camera);
    if (!cameraIds(env).includes(camera)) invalid('Choose an available camera.');
    const rows = await ops.recordings(
      env,
      camera,
      window.from,
      window.until,
      new Date(Date.now() - RETENTION_MS).toISOString(),
      options.since === undefined ? undefined : { after: options.since ?? '', until: created },
    );
    if (mode === 'run' && rows.length > 100) invalid('Use a smaller window: one run can contain up to 100 recordings.');
    for (const row of rows) {
      if (mode === 'test' && estimated >= limit) break;
      if (!hash(row.sha256) || !row.duration_s || row.duration_s > 300)
        invalid('A recording has incomplete metadata or exceeds five minutes.');
      const from = Math.max(0, (Date.parse(window.from) - Date.parse(row.start)) / 1000),
        until = Math.min(row.duration_s, (Date.parse(window.until) - Date.parse(row.start)) / 1000);
      // Whole slots only. A partial last slot can end before the recording's last frame starts:
      // a 57.03 s clip at 8 fps has no frame at 57 s, and the processor would drop the whole clip.
      const slots = Math.floor((until - from) / interval);
      if (slots < 1) continue;
      const sampleLimit = mode === 'test' ? Math.min(limit - estimated, slots) : slots;
      estimated += sampleLimit;
      const source: WorkshopInput = {
        kind: 'video',
        sample_limit: sampleLimit,
        recording: row,
        item: null,
        from,
        until,
      };
      seeds.push({
        id: `${body.id}_${seeds.length}`,
        expected_images: sampleLimit,
        cam: camera,
        input_json: JSON.stringify(source),
        expires_at: new Date(
          Math.min(Date.parse(row.start) + RETENTION_MS, mode === 'test' ? Date.parse(previewExpiry) : Infinity),
        ).toISOString(),
      });
    }
  } else {
    const items = await ops.inputItems(
      env,
      String(input.data.bucketId),
      created,
      mode === 'test' ? limit : 2001,
      options.since === undefined ? '' : (options.since ?? ''),
      input.data.includeRejected === true,
    );
    if (items.length > 2000) invalid('This runner accepts up to 2,000 labeled input images per run.');
    for (const item of items) {
      const source: WorkshopInput = {
        kind: 'image',
        recording: null,
        item,
        from: 0,
        until: 0,
      };
      seeds.push({
        id: `${body.id}_${seeds.length}`,
        expected_images: 1,
        cam: item.cam,
        input_json: JSON.stringify(source),
        expires_at: mode === 'test' && previewExpiry < item.expires_at ? previewExpiry : item.expires_at,
      });
    }
    estimated = items.length;
  }
  if (!seeds.length) invalid(noMedia);
  if (estimated > 2000)
    invalid('This window exceeds 2,000 sampled images. Increase the sampling interval or choose a smaller window.');
  snapshot.expected_images = estimated;
  const r = {
    id: String(body.id),
    request_hash: requestHash,
    workflow_id: workflow.id,
    name: options.name ?? workflow.name,
    runtime_id: runtime.id,
    snapshot_json: canonicalModel(snapshot),
    created_at: created,
  };
  await ops.insertRun(env, r, seeds, state.revision);
  const saved = await ops.run(env, r.id);
  if (!saved)
    throw new ApplicationError('conflict', 'Workshop changed while preparing the run. Reload before running.');
  if (saved.request_hash !== requestHash) throw new ApplicationError('conflict', 'Run ID conflict.');
  return saved;
}
/** Why this runner cannot use a model version, or null when it can. */
export function workshopModelIssue(model: ModelVersion) {
  if (
    model.adapter !== 'ultralytics' ||
    !['object-detection', 'image-classification'].includes(model.task) ||
    !model.classes.length
  )
    return `${model.name} needs a verified runtime adapter or class mapping.`;
  // Crop-trained classifiers and tile-trained detectors need explicit preprocessing nodes.
  if (
    /tiled|crop/i.test(model.name) ||
    model.input.kind === 'crop' ||
    (record(model.input.legacy_runtime) && model.input.legacy_runtime.prefers_tiling === true)
  )
    return `${model.name} requires tiling or crops. Choose a full-frame model for this runner.`;
  if (!model.artifacts.some(a => a.filename.endsWith('.pt'))) return `The weights for ${model.name} are unavailable.`;
  return null;
}
// Not a valid workflow ID, so no saved workflow can share it.
export const photoWorkflowId = '@photo';
/** Workflow runs, newest first, optionally for one workflow. Agent photos are runs too, but they are not shown with workflows. */
export const listRuns = (env: Env, workflow: string | null = null) =>
  ops.listRuns(env, photoWorkflowId, now(), { workflow });
export async function cancelRun(env: Env, id: string) {
  if (!(await ops.run(env, id))) throw new ApplicationError('not_found', 'Run not found.');
  await ops.cancel(env, id);
  await publishRun(env, id);
  return { cancelled: true };
}
export async function bucketContents(env: Env, id: string) {
  if (!(await loadWorkshop(env)).document.buckets.some(b => b.id === id))
    throw new ApplicationError('not_found', 'Bucket not found.');
  const rows = await ops.items(env, id, now());
  return {
    items: rows.slice(0, 500).map(publicItem),
    has_more: rows.length > 500,
    accepted: (await ops.bucketCounts(env, now())).find(bucket => bucket.bucket_id === id)?.accepted ?? 0,
    versions: await ops.versions(env, id, now()),
  };
}
function publicItem(i: WorkshopItem) {
  return {
    ...i,
    annotations: JSON.parse(i.annotation_json),
    image_url: `/api/workshop/items/${i.id}/image`,
  };
}
/** How long a browser may keep a photo or its thumbnail: a day, and never past the photo's expiry. */
export const browserCacheSeconds = (expiresAt: string) =>
  Math.max(0, Math.min(86400, Math.floor((Date.parse(expiresAt) - Date.now()) / 1000)));

/** A photo for the website, with how long the browser may keep it. Photo bytes never change. */
export async function websiteImage(env: Env, id: string) {
  const item = await ops.item(env, id, now());
  const image = item && (await ops.image(env, item.object_key));
  if (!item || !image) throw new ApplicationError('not_found', 'Image expired or missing.');
  return { body: image.body, maxAge: browserCacheSeconds(item.expires_at) };
}

export async function readImage(env: Env, id: string) {
  const item = await ops.item(env, id, now());
  if (!item) throw new ApplicationError('not_found', 'Image expired or missing.');
  const image = await ops.image(env, item.object_key);
  if (!image) throw new ApplicationError('not_found', 'Image bytes are no longer available.');
  return image;
}
export async function createVersion(env: Env, bucket: string, body: JsonObject) {
  if (!identifier(body.id) || typeof body.name !== 'string' || !body.name.trim() || body.name.length > 100)
    invalid('Provide a version ID and name.');
  if (body.include_rejected !== undefined && typeof body.include_rejected !== 'boolean')
    invalid('include_rejected must be true or false.');
  const previous = await ops.version(env, body.id);
  if (previous) {
    if (previous.bucket_id !== bucket || previous.name !== body.name.trim())
      throw new ApplicationError('conflict', 'Version ID conflict.');
    const { manifest, available, ...metadata } = await readVersion(env, body.id);
    return { ...metadata, images: manifest.items.length };
  }
  const assets = new Set<string>(),
    selected = (await ops.reviewedItems(env, bucket, now(), body.include_rejected === true)).filter(i => {
      if (assets.has(i.asset_id)) return false;
      assets.add(i.asset_id);
      return true;
    });
  if (!selected.length) invalid('Accept some labeled images before creating a version.');
  // No per-image storage check: cleanup never deletes bytes a live item references, and a check per image
  // took over five minutes from Singapore for 7,471 images (each a round trip to storage in eastern North America).
  const entries: WorkshopVersionManifest['items'] = [];
  for (const i of selected) {
    const group = parseInt((await sha256(`${i.cam}/${i.captured_at.slice(0, 10)}`)).slice(0, 8), 16) % 100;
    entries.push({
      item_id: i.id,
      cam: i.cam,
      media_path: `/v1/workshop/items/${i.id}/image`,
      asset_id: i.asset_id,
      sha256: i.sha256,
      source_sha256: i.source_sha256,
      captured_at: i.captured_at,
      expires_at: i.expires_at,
      review: i.review as 'accepted' | 'rejected',
      annotations: JSON.parse(i.annotation_json),
      provenance: JSON.parse(i.provenance_json),
      split: group < 80 ? 'train' : group < 90 ? 'validation' : 'test',
    });
  }
  const tasks = new Set(entries.map(e => (e.provenance.annotation_model as { task: string }).task));
  if (tasks.size !== 1)
    invalid('A dataset version needs one task. Separate detection and classification outputs into different buckets.');
  const classes = [
    ...new Set(
      entries.flatMap(e => (e.provenance.annotation_model as { classes: { name: string }[] }).classes.map(c => c.name)),
    ),
  ].sort();
  for (const entry of entries)
    entry.annotations = entry.annotations.map(a => ({
      ...a,
      model_class_id: a.class_id,
      class_id: classes.indexOf(a.class_name),
    }));
  const created = now(),
    manifest: WorkshopVersionManifest = {
      schema_version: 1,
      bucket_id: bucket,
      class_names: classes,
      split_policy:
        'camera + UTC capture day; SHA-256 modulo 100; 80/10/10. Entire days stay together; small versions may contain only one split.',
      created_at: created,
      items: entries,
    };
  const { items, ...header } = manifest,
    v = {
      id: body.id,
      bucket_id: bucket,
      name: body.name.trim(),
      manifest_json: canonicalModel(header),
      sha256: await sha256(canonicalModel(manifest)),
      created_at: created,
    };
  if (
    !(await ops.insertVersion(env, v, items, now()))
  )
    throw new ApplicationError('conflict', 'Images changed or expired during version creation. Refresh and try again.');
  const { manifest_json, ...metadata } = v;
  return { ...metadata, images: items.length };
}
export async function readVersion(env: Env, id: string) {
  const v = await ops.versionManifest(env, id);
  // Every version has an item; none means a row written without its items, which is not a dataset.
  if (!v?.items.length) throw new ApplicationError('not_found', 'Version not found.');
  const manifest = {
    ...JSON.parse(v.manifest_json),
    items: v.items.map(i => JSON.parse(i)),
  } as WorkshopVersionManifest;
  if (manifest.items.some(i => i.expires_at <= now()))
    throw new ApplicationError('not_found', 'This dataset version has expired with its source images.');
  const { manifest_json, items, ...metadata } = v;
  return { ...metadata, manifest, available: true };
}
export async function cleanupWorkshop(env: Env) {
  if (!env.WORKSHOP) return;
  for (const upload of await ops.expiredUploads(env, now())) await ops.removeUpload(env, upload.object_key);
  await ops.purgeExpiredLabels(env, now());
}
/** Save a verified frame, labels and provenance; the source video keeps its original lifetime. */
export async function saveItem(env: Env, id: string) {
  const item = await ops.item(env, id, now());
  if (!item) throw new ApplicationError('not_found', 'Image expired or missing.');
  if (item.object_key.startsWith('retained/')) return { saved: true };
  const source = await ops.image(env, item.object_key);
  if (!source) throw new ApplicationError('not_found', 'Source image is no longer available.');
  const objectKey = `retained/${crypto.randomUUID()}/${item.sha256}.jpg`;
  const attemptExpiry = new Date(Date.now() + 3600000).toISOString();
  await ops.upload(env, {
    object_key: objectKey,
    job_id: item.job_id,
    generation: 0,
    sha256: item.sha256,
    bytes: source.size,
    expires_at: attemptExpiry,
  });
  await ops.copyRetained(env, objectKey, source.body, item.sha256);
  const verified = await ops.imageHead(env, objectKey);
  if (!verified || verified.size !== source.size)
    throw new ApplicationError('unavailable', 'Retained image verification failed.');
  const provenance = {
    ...JSON.parse(item.provenance_json),
    saved_at: now(),
    source_expires_at: item.expires_at,
    retention_scope: 'image, labels, and provenance; original video not included',
  };
  if (!(await ops.retainItem(env, item, objectKey, canonicalModel(provenance), '9999-12-31T23:59:59.999Z', now()))) {
    const latest = await ops.item(env, id, now());
    if (!latest?.object_key.startsWith('retained/'))
      throw new ApplicationError('conflict', 'Image expired or changed before it could be saved.');
  }
  return { saved: true };
}

export async function machineVersion(env: Env, principal: ApiPrincipal, id: string) {
  if (principal.scope !== 'footage:read' && principal.scope !== 'agent')
    throw new ApplicationError('forbidden', 'Use a read-only data key.');
  const version = await readVersion(env, id);
  if (version.manifest.items.some(i => !principal.cameras.includes(i.cam)))
    throw new ApplicationError('forbidden', 'This key cannot access every camera in this version.');
  return version;
}
export async function machineImage(env: Env, principal: ApiPrincipal, id: string) {
  if (principal.scope !== 'footage:read' && principal.scope !== 'agent')
    throw new ApplicationError('forbidden', 'Use a read-only data key.');
  const item = await ops.item(env, id, now());
  if (!item) throw new ApplicationError('not_found', 'Image expired or missing.');
  if (!principal.cameras.includes(item.cam))
    throw new ApplicationError('forbidden', 'This key cannot access that camera.');
  return readImage(env, id);
}