Skip to content

armgit / bygga

Public Imported from GitHub
0

IKEA assembly, step by step: animated, manual-style 3D guides. Scan any paper manual with Together AI.

TypeScript
7.1 KB
Raw Blame
import { buildPlan } from "@/lib/plan";
import { clean, fallbackSpec, parseScan } from "@/lib/scan/parse";
import { PAGE_PROMPT, PLAN_PROMPT, planInput, repairInput } from "@/lib/scan/prompts";
import { SAMPLE_SCAN } from "@/lib/scan/sample";
import { chatText, extractJson, fromApp, ipOf, isDevKey, MODELS, rateLimited, TogetherError, userKey, type Msg } from "@/lib/together";
import type { ProductSpec } from "@/lib/types";
import { SCAN_ENABLED } from "@/lib/site";

export const maxDuration = 300;

type Note = { kind?: string; product?: { name?: string; brand?: string; type?: string }; parts?: unknown[]; hardware?: unknown[]; steps?: { n?: number }[] };

export async function POST(req: Request) {
  if (!SCAN_ENABLED) return Response.json({ error: "off" }, { status: 404 });
  if (!fromApp(req)) return Response.json({ error: "forbidden" }, { status: 403 });
  // the dev key tries the scan flow locally without spending credits
  const mock = isDevKey(req);
  const key = mock ? "mock" : userKey(req);
  if (!key) return Response.json({ error: "key" }, { status: 401 });
  if (rateLimited(ipOf(req) + ":scan", 6, 10 * 60_000)) return Response.json({ error: "busy" }, { status: 429 });
  const body = (await req.json().catch(() => null)) as { pages?: unknown[]; hint?: unknown } | null;
  const pages = (body?.pages ?? []).filter((p): p is string => typeof p === "string" && p.startsWith("data:image/") && p.length < 4_000_000).slice(0, 12);
  if (!pages.length) return Response.json({ error: "pages" }, { status: 400 });
  const hint = clean(String(body?.hint ?? "")).slice(0, 80);

  const enc = new TextEncoder();
  const stream = new ReadableStream({
    async start(ctrl) {
      let open = true;
      const send = (o: object) => {
        if (open) ctrl.enqueue(enc.encode(JSON.stringify(o) + "\n"));
      };
      try {
        send({ type: "stage", stage: "reading", total: pages.length });
        const spec = mock ? await fake(pages.length, hint, send) : await scan(key, pages, hint, send, req.signal);
        send({ type: "done", spec });
      } catch (e) {
        const err = e as TogetherError;
        const status = err instanceof TogetherError ? err.status : 0;
        console.error("[scan]", status, (err.message ?? "").slice(0, 200));
        const code = status === 401 || status === 403 ? "key" : status === 402 ? "credits" : status === 429 ? "busy" : (err.message ?? "").startsWith("unreadable") ? "unreadable" : "failed";
        send({ type: "error", code });
      } finally {
        open = false;
        ctrl.close();
      }
    },
  });
  return new Response(stream, { headers: { "Content-Type": "application/x-ndjson; charset=utf-8", "Cache-Control": "no-store" } });
}

async function scan(key: string, pages: string[], hint: string, send: (o: object) => void, signal: AbortSignal): Promise<ProductSpec> {
  // 1. read every page on its own, a few at a time
  const notes: Note[] = new Array(pages.length);
  let next = 0;
  let read = 0;
  const worker = async () => {
    while (next < pages.length) {
      const i = next++;
      try {
        const { text } = await chatText({
          key,
          models: MODELS.read,
          maxTokens: 2000,
          temperature: 0,
          signal,
          messages: [{ role: "user", content: [{ type: "text", text: PAGE_PROMPT }, { type: "image_url", image_url: { url: pages[i] } }] }],
        });
        notes[i] = (extractJson(text) as Note | null) ?? { kind: "other" };
      } catch (e) {
        if (e instanceof TogetherError && [401, 402, 403, 429].includes(e.status)) throw e;
        notes[i] = { kind: "other" };
      }
      read++;
      send({ type: "page", index: i, read, total: pages.length, label: pageLabel(notes[i]), product: notes[i].product?.name ? clean(String(notes[i].product?.name)).slice(0, 40) : undefined, brand: notes[i].product?.brand ? clean(String(notes[i].product?.brand)).slice(0, 40) : undefined });
    }
  };
  await Promise.all(Array.from({ length: Math.min(4, pages.length) }, worker));
  const useful = notes.filter((n) => n && n.kind !== "other");
  if (!useful.length && !hint) throw new Error("unreadable");

  // 2. plan the build
  send({ type: "stage", stage: "planning" });
  const msgs: Msg[] = [
    { role: "system", content: PLAN_PROMPT },
    { role: "user", content: planInput(notes, hint) },
  ];
  const ask = async () => {
    const { text } = await chatText({ key, models: MODELS.plan, json: true, maxTokens: 16000, temperature: 0.2, reasoning: "medium", signal, messages: msgs });
    return { text, raw: extractJson(text) };
  };
  let { text, raw } = await ask();
  let r = parseScan(raw);
  if (r.spec) send({ type: "drafted", name: r.spec.name, brand: r.spec.brand, parts: r.spec.custom?.parts.length ?? 0, steps: r.spec.custom?.steps.length ?? 0 });

  // 3. check it fits together, ask for one round of fixes if not
  send({ type: "stage", stage: "checking" });
  let ok = !!r.spec && r.check.ok && builds(r.spec);
  if (!ok && raw) {
    send({ type: "stage", stage: "fixing" });
    msgs.push({ role: "assistant", content: text.slice(0, 24000) }, { role: "user", content: repairInput(r.check.issues.length ? r.check.issues : ["The parts list or steps couldn't be read. Make sure every part has id, name, size and pos, and every step has a title."]) });
    ({ text, raw } = await ask());
    const again = parseScan(raw);
    if (again.spec && again.check.ok && builds(again.spec)) {
      r = again;
      ok = true;
    }
  }
  if (ok && r.spec) return r.spec;
  const fb = fallbackSpec(raw);
  if (fb && builds(fb)) return fb;
  throw new Error("unbuildable");
}

function builds(spec: ProductSpec) {
  try {
    const p = buildPlan(spec);
    return p.steps.length > 0 && p.parts.length > 0;
  } catch {
    return false;
  }
}

function pageLabel(n: Note) {
  const steps = (n.steps ?? []).map((s) => Number(s?.n)).filter((x) => Number.isFinite(x) && x > 0);
  if (steps.length) {
    const a = Math.min(...steps), b = Math.max(...steps);
    return a === b ? `Step ${a}` : `Steps ${a} to ${b}`;
  }
  switch (n.kind) {
    case "cover":
      return "Cover";
    case "parts":
      return "Parts list";
    case "hardware":
      return "Fittings";
    case "tools":
      return "Tools";
    case "warnings":
      return "Safety notes";
    default:
      return "Page";
  }
}

async function fake(total: number, hint: string, send: (o: object) => void): Promise<ProductSpec> {
  const wait = (ms: number) => new Promise((r) => setTimeout(r, ms));
  const labels = ["Cover", "Parts list", "Fittings", "Steps 1 to 3", "Steps 4 to 6", "Steps 7 to 10"];
  for (let i = 0; i < total; i++) {
    await wait(700);
    send({ type: "page", index: i, read: i + 1, total, label: labels[i] ?? `Steps ${i * 2} to ${i * 2 + 1}`, product: i === 0 ? SAMPLE_SCAN.name : undefined, brand: i === 0 ? SAMPLE_SCAN.brand : undefined });
  }
  send({ type: "stage", stage: "planning" });
  await wait(2200);
  const spec: ProductSpec = { ...SAMPLE_SCAN, name: hint ? hint.split(" ")[0].toUpperCase().slice(0, 24) : SAMPLE_SCAN.name };
  send({ type: "drafted", name: spec.name, brand: spec.brand, parts: spec.custom!.parts.length, steps: spec.custom!.steps.length });
  send({ type: "stage", stage: "checking" });
  await wait(900);
  return spec;
}