Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

7.0 KB
Raw Blame
import type { DecisionRequest, SystemOneRequest } from "./schemas";
import type { OpenAIDecisionsRequest } from "./openAIDecisions";
import { parseJson } from "./validatePayload";
import { blankQuestion } from "./formPayload";

type PresetChoice = {
  type: "choice";
  instructions: string;
  criteria: Record<string, string>;
};

type PresetNoul = {
  type: "noul";
  instructions: string;
  criteria?: { true: string; false: string };
};

type PresetScore = {
  type: "score";
  instructions: string;
  criteria: string[];
};

export type PresetQuestion = PresetChoice | PresetNoul | PresetScore;

export type PresetRequest = {
  state: string;
  questions: Record<string, PresetQuestion>;
};

type PresetEndpoint = "/v1/decisions" | "/v1/systemone";

export interface DecisionPreset {
  id: string;
  label: string;
  request: PresetRequest;
}

export const SYSTEM_ONE_EXAMPLE = {
  model: "jev-latest",
  state:
    "Since upgrading to the latest release, streaming responses stop halfway through whenever a fallback model takes over. Non-streaming requests still work. I haven't narrowed down which change caused it, but it happens on most long prompts.",
  questions: {
    area: {
      type: "choice",
      instructions: "Which part of the project does this issue belong to?",
      criteria: {
        backend: "Server, API routing, streaming, and fallbacks",
        sdk: "The client library and its helpers",
        ui: "The web dashboard",
        docs: "Documentation and examples",
      },
    },
    has_repro_steps: {
      type: "noul",
      instructions: "Does the issue include steps someone could follow to reproduce it?",
      criteria: {
        true: "It gives concrete steps, a config, or a request that triggers the bug",
        false: "It only describes the symptom without a way to trigger it",
      },
    },
    severity: {
      type: "score",
      instructions: "How severe is this issue?",
      criteria: [
        "Cosmetic or a typo",
        "Minor bug with an easy workaround",
        "Broken feature with a workaround",
        "Broken feature with no workaround",
        "Outage or data loss",
      ],
    },
  },
} satisfies SystemOneRequest;

const { model: _exampleModel, ...bugTriage }: SystemOneRequest & PresetRequest = SYSTEM_ONE_EXAMPLE;

const supportRouting = {
  state:
    "Hi, I was charged twice for my team plan this month and the second charge pushed my card over its limit. I need the duplicate refunded today and want to know why it happened.",
  questions: {
    department: {
      type: "choice",
      instructions: "Which team should handle this ticket?",
      criteria: {
        billing: "Invoices, charges, refunds, and plan changes",
        technical: "Bugs, errors, outages, and integration help",
        account: "Login, permissions, and profile settings",
        sales: "Pricing questions and new purchases",
      },
    },
  },
} satisfies PresetRequest;

const moderation = {
  state:
    "This guide is garbage and whoever wrote it should be fired. The fix that actually works is setting stream=false, took me two days to figure that out.",
  questions: {
    breaks_rules: {
      type: "noul",
      instructions: "Does this comment break the community guidelines?",
      criteria: {
        true: "Insults, harassment, slurs, or threats aimed at a person",
        false: "Blunt or frustrated but still about the content",
      },
    },
  },
} satisfies PresetRequest;

const leadScoring = {
  state:
    "Inbound form: VP Engineering at a 400 person fintech. Running a self hosted gateway today, evaluating replacements this quarter, budget approved, needs SSO and audit logs.",
  questions: {
    fit: {
      type: "score",
      instructions: "How well does this lead fit our ideal customer?",
      criteria: [
        "No fit, wrong market or no budget",
        "Weak fit, early curiosity only",
        "Possible fit, needs discovery",
        "Strong fit, active evaluation",
        "Ideal fit, ready to buy this quarter",
      ],
    },
  },
} satisfies PresetRequest;

export const DECISION_PRESETS: readonly DecisionPreset[] = [
  { id: "bug-triage", label: "Triage a bug report", request: bugTriage },
  { id: "support-routing", label: "Route a support ticket", request: supportRouting },
  { id: "moderation", label: "Moderate a forum comment", request: moderation },
  { id: "lead-scoring", label: "Score a sales lead", request: leadScoring },
];

const toOpenAIQuestion = (name: string, question: PresetQuestion): OpenAIDecisionsRequest["questions"][number] => {
  switch (question.type) {
    case "choice":
      return {
        type: "choice",
        name,
        instructions: question.instructions,
        choices: Object.entries(question.criteria).map(([value, description]) => ({ value, description })),
      };
    case "noul":
      return { type: "predicate", name, instructions: question.instructions };
    case "score":
      return {
        type: "score",
        name,
        instructions: question.instructions,
        levels: question.criteria.map((description, index) => ({ label: String(index), description })),
      };
  }
};

export const toSystemOneRequest = (request: PresetRequest, model?: string): DecisionRequest => ({ model, ...request });

export const toOpenAIDecisionsRequest = (request: PresetRequest, model?: string): OpenAIDecisionsRequest => ({
  model,
  input: request.state,
  questions: Object.entries(request.questions).map(([name, question]) => toOpenAIQuestion(name, question)),
});

export const presetPayload = (preset: DecisionPreset, endpoint: PresetEndpoint, model?: string): string =>
  JSON.stringify(
    endpoint === "/v1/decisions"
      ? toOpenAIDecisionsRequest(preset.request, model)
      : toSystemOneRequest(preset.request, model),
    null,
    2,
  );

const stableJson = (value: unknown): string =>
  JSON.stringify(value, (_key, item: unknown) =>
    item !== null && typeof item === "object" && !Array.isArray(item)
      ? Object.fromEntries(Object.entries(item as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b)))
      : item,
  );

const withoutModel = (value: unknown): string => {
  const { model: _model, ...rest } = (value ?? {}) as Record<string, unknown>;
  return stableJson(rest);
};

export const matchPreset = (endpoint: PresetEndpoint, raw: string): DecisionPreset | undefined => {
  const json = parseJson(raw);
  if (!json.ok) {
    return undefined;
  }
  const current = withoutModel(json.value);
  return DECISION_PRESETS.find((preset) => withoutModel(JSON.parse(presetPayload(preset, endpoint))) === current);
};

export const emptyPayload = (endpoint: PresetEndpoint, model?: string): string =>
  JSON.stringify(
    endpoint === "/v1/decisions"
      ? { model, input: "", questions: [{ type: "predicate", name: "", instructions: "" }] }
      : { model, state: "", questions: { "": blankQuestion() } },
    null,
    2,
  );

export const decisionsExample = (model?: string): DecisionRequest => toSystemOneRequest(bugTriage, model);

export const openAIDecisionsExample = (model?: string): OpenAIDecisionsRequest =>
  toOpenAIDecisionsRequest(bugTriage, model);