Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
4.0 KB
Raw Blame
import type { Usage } from "../gen";
import { ApiError } from "../bridge/frames";

export interface SseEvent {
  data: string;
}

export function splitSseEvents(buf: string): { events: SseEvent[]; rest: string } {
  const events: SseEvent[] = [];
  let rest = buf;
  for (;;) {
    const m = rest.match(/\r?\n\r?\n/);
    if (!m || m.index === undefined) break;
    const block = rest.slice(0, m.index);
    rest = rest.slice(m.index + m[0].length);
    const dataLines: string[] = [];
    for (const line of block.split(/\r?\n/)) {
      if (line.startsWith("data:")) {
        dataLines.push(line.slice(5).trimStart());
      }
    }
    if (dataLines.length > 0) events.push({ data: dataLines.join("\n") });
  }
  return { events, rest };
}

/** Keep `delta.content` and `delta.reasoning_content` on separate sinks so reasoning never enters the visible body. */
export interface ChatDeltaSink {
  onText(text: string): void;
  onReasoning(text: string): void;
  onUsage(usage: Usage, perf?: PerfEdge0): void;
}

export interface PerfEdge0 {
  ttft_ms?: number;
  prefill_tps?: number;
  decode_tps?: number;
  footprint_mb?: number;
}

export async function consumeChatSse(
  res: Response,
  sink: ChatDeltaSink,
  signal?: AbortSignal,
): Promise<void> {
  const reader = res.body?.getReader();
  if (!reader) throw new ApiError(res.status, null, "streaming response has no body");
  let aborted = false;
  const onAbort = () => {
    aborted = true;
    void reader.cancel();
  };
  signal?.addEventListener("abort", onAbort, { once: true });
  const decoder = new TextDecoder();
  let buf = "";
  let doneFlagged = false;
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    buf += decoder.decode(value, { stream: true });
    const { events, rest } = splitSseEvents(buf);
    buf = rest;
    for (const ev of events) {
      if (ev.data === "[DONE]") {
        doneFlagged = true;
        continue;
      }
      let obj: {
        choices?: { delta?: { content?: string; reasoning_content?: string }; finish_reason?: string | null }[];
        usage?: {
          prompt_tokens?: number;
          completion_tokens?: number;
          total_tokens?: number;
          edge0?: Record<string, unknown> | null;
        } | null;
        error?: { code?: string; message?: string };
      };
      try {
        obj = JSON.parse(ev.data);
      } catch {
        continue;
      }
      if (obj.error) {
        throw new ApiError(res.status, obj.error.code ?? "E-SRV-WORKER", obj.error.message ?? "in-stream error");
      }
      const delta = obj.choices?.[0]?.delta;
      if (delta?.content) sink.onText(delta.content);
      if (delta?.reasoning_content) sink.onReasoning(delta.reasoning_content);
      if (obj.usage && typeof obj.usage.total_tokens === "number") {
        const e = obj.usage.edge0 ?? {};
        const num = (k: string): number | undefined =>
          typeof e[k] === "number" ? (e[k] as number) : undefined;
        const perf: PerfEdge0 = {
          ttft_ms: num("ttft_ms"),
          prefill_tps: num("prefill_tps"),
          decode_tps: num("decode_tps"),
          footprint_mb: num("footprint_mb"),
        };
        sink.onUsage(
          {
            prompt_tokens: obj.usage.prompt_tokens ?? 0,
            completion_tokens: obj.usage.completion_tokens ?? 0,
            total_tokens: obj.usage.total_tokens,
          },
          perf,
        );
      }
    }
  }
  signal?.removeEventListener("abort", onAbort);
  if (aborted) return;
  if (!doneFlagged && buf.trim().length > 0) {
    throw new ApiError(res.status, "E-SRV-WORKER", "stream ended without [DONE] (connection truncated)");
  }
}

export async function parseErrorEnvelope(res: Response): Promise<ApiError> {
  let code: string | null = null;
  let message = `HTTP ${res.status}`;
  try {
    const j = (await res.json()) as { error?: { code?: string; message?: string } };
    code = j?.error?.code ?? null;
    message = j?.error?.message ?? message;
  } catch {
  }
  return new ApiError(res.status, code, message);
}