A desktop coding agent with free access to frontier models.
TypeScript
import assert from "node:assert/strict"
import { test } from "node:test"
import { createDenTypeId } from "@omnirush-ee/utils/typeid"
import { Hono } from "hono"
import type { OpenRouterUnknownModelUsageReport } from "../src/webhooks.js"
process.env.OMNIRUSH_DEV_MODE = "1"
process.env.DATABASE_URL = "mysql://root:[email protected]:3306/omnirush_den"
process.env.DEN_DB_ENCRYPTION_KEY = "local-dev-db-encryption-key-please-change-1234567890"
process.env.INFERENCE_WEBHOOK_SECRET = "local-dev-webhook-secret"
const { registerWebhookRoutes } = await import("../src/webhooks.js")
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
}
function attribute(key: string, value: string | number | boolean) {
if (typeof value === "string") return { key, value: { stringValue: value } }
if (typeof value === "boolean") return { key, value: { boolValue: value } }
if (Number.isInteger(value)) return { key, value: { intValue: value } }
return { key, value: { doubleValue: value } }
}
function webhookRequest(body: unknown) {
return new Request("http://omnirush.test/webhooks/openrouter", {
method: "POST",
headers: {
authorization: "Bearer local-dev-webhook-secret",
"content-type": "application/json",
},
body: JSON.stringify(body),
})
}
async function responseJson(response: Response) {
const payload: unknown = await response.json()
assert.ok(isRecord(payload))
return payload
}
function createWebhookTestServer() {
const app = new Hono()
const organizationId = createDenTypeId("organization")
const orgMembershipId = createDenTypeId("member")
const inferenceKeyId = createDenTypeId("inferenceKey")
const reports: OpenRouterUnknownModelUsageReport[] = []
const insertedEntries: {
omnirushRequestId: string
externalEventId: string | null
costAmount: number | null
modelId: string
providerId: string
inputTokens: number | null
outputTokens: number | null
totalTokens: number | null
}[] = []
const bucketCharges: { amount: number }[] = []
const calls = {
settleUsage: 0,
}
registerWebhookRoutes(app, {
reporter: {
unknownModel(report) {
reports.push(report)
},
},
async findInferenceKey(_inferenceKeyId) {
return {
id: inferenceKeyId,
status: "active",
revoked_at: null,
organization_id: organizationId,
org_membership_id: orgMembershipId,
}
},
async settleUsage(input) {
calls.settleUsage += 1
insertedEntries.push({
omnirushRequestId: input.span.omnirushRequestId,
externalEventId: input.span.externalEventId,
costAmount: input.costAmount,
modelId: input.span.reportedModel,
providerId: "openrouter",
inputTokens: input.span.usageMetadata.inputTokens,
outputTokens: input.span.usageMetadata.outputTokens,
totalTokens: input.span.usageMetadata.totalTokens,
})
if (input.costAmount === null) return "deferred"
bucketCharges.push({ amount: input.costAmount })
return "ingested"
},
})
function usagePayload(input: { requestId: string; eventId: string; generationId: string; requestModel: string; responseModel: string; includeSensitive?: boolean }) {
const attributes = [
attribute("trace.org_membership_id", orgMembershipId),
attribute("trace.inference_key_id", inferenceKeyId),
attribute("trace.omnirush_request_id", input.requestId),
attribute("event_id", input.eventId),
attribute("gen_ai.response.id", input.generationId),
attribute("gen_ai.request.model", input.requestModel),
attribute("gen_ai.response.model", input.responseModel),
attribute("gen_ai.usage.input_cost", 0),
attribute("gen_ai.usage.output_cost", 0),
attribute("gen_ai.usage.input_tokens", 11),
attribute("gen_ai.usage.output_tokens", 13),
attribute("gen_ai.usage.total_tokens", 24),
attribute("gen_ai.usage.currency", "USD"),
]
if (input.includeSensitive) {
attributes.push(
attribute("authorization", "Bearer caller-secret"),
attribute("gen_ai.prompt.0.content", "secret prompt content"),
attribute("gen_ai.completion.0.content", "secret response content"),
)
}
return {
resourceSpans: [{
resource: { attributes: [] },
scopeSpans: [{
scope: { attributes: [] },
spans: [{
traceId: "trace-123",
spanId: "span-123",
name: "OpenRouter usage",
startTimeUnixNano: "1700000000000000000",
endTimeUnixNano: "1700000001000000000",
attributes,
}],
}],
}],
}
}
return { app, reports, insertedEntries, bucketCharges, calls, organizationId, usagePayload }
}
test("retains unknown model usage without deduction and reports bounded provider facts", async () => {
const { app, reports, insertedEntries, bucketCharges, calls, organizationId, usagePayload } = createWebhookTestServer()
const response = await app.fetch(webhookRequest(usagePayload({
requestId: "request-unknown",
eventId: "event-unknown",
generationId: "generation-unknown",
requestModel: "z-ai/glm-5.2",
responseModel: "vendor/new-model",
includeSensitive: true,
})))
assert.equal(response.status, 200)
const payload = await responseJson(response)
assert.equal(payload.ingested, 0)
assert.equal(payload.skipped, 0)
assert.equal(payload.deferred, 1)
assert.equal(calls.settleUsage, 1)
assert.equal(insertedEntries.length, 1)
assert.equal(insertedEntries[0]?.costAmount, null)
assert.equal(bucketCharges.length, 0)
assert.equal(reports.length, 1)
const report = reports[0]
assert.ok(report)
assert.equal(report.reportedModel, "vendor/new-model")
assert.equal(report.organizationId, organizationId)
assert.equal(report.omnirushRequestId, "request-unknown")
assert.equal(report.externalEventId, "event-unknown")
assert.equal(report.generationId, "generation-unknown")
assert.equal(report.usage.requestModel, "z-ai/glm-5.2")
assert.equal(report.usage.responseModel, "vendor/new-model")
assert.equal(report.usage.inputTokens, 11)
assert.equal(report.usage.outputTokens, 13)
assert.equal(report.usage.totalTokens, 24)
assert.equal(report.usage.currency, "USD")
const diagnosticText = JSON.stringify(report)
assert.ok(!diagnosticText.includes("caller-secret"))
assert.ok(!diagnosticText.includes("secret prompt content"))
assert.ok(!diagnosticText.includes("secret response content"))
})
test("deducts usage without Sentry diagnostics when OpenRouter usage reports a known model", async () => {
const { app, reports, insertedEntries, bucketCharges, calls, usagePayload } = createWebhookTestServer()
const response = await app.fetch(webhookRequest(usagePayload({
requestId: "request-known",
eventId: "event-known",
generationId: "generation-known",
requestModel: "z-ai/glm-5.2",
responseModel: "z-ai/glm-5.2",
})))
assert.equal(response.status, 200)
const payload = await responseJson(response)
assert.equal(payload.ingested, 1)
assert.equal(payload.skipped, 0)
assert.equal(reports.length, 0)
assert.equal(calls.settleUsage, 1)
assert.deepEqual(insertedEntries, [{
omnirushRequestId: "request-known",
externalEventId: "event-known",
costAmount: 1,
modelId: "z-ai/glm-5.2",
providerId: "openrouter",
inputTokens: 11,
outputTokens: 13,
totalTokens: 24,
}])
assert.deepEqual(bucketCharges, [{ amount: 1 }])
})
test("blank provider prices remain unpriced rather than fabricated zero charges", async () => {
const { app, calls, insertedEntries, bucketCharges, usagePayload } = createWebhookTestServer()
const payload = usagePayload({ requestId: "blank-price", eventId: "blank-price", generationId: "blank-price", requestModel: "z-ai/glm-5.2", responseModel: "z-ai/glm-5.2" })
const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!
span.attributes = span.attributes.map((attr) => attr.key === "gen_ai.usage.input_cost" || attr.key === "gen_ai.usage.output_cost" ? attribute(attr.key, " ") : attr)
const response = await app.fetch(webhookRequest(payload))
assert.equal(response.status, 200)
assert.equal((await responseJson(response)).deferred, 1)
assert.equal(calls.settleUsage, 1)
assert.equal(insertedEntries[0]?.costAmount, null)
assert.equal(bucketCharges.length, 0)
})
test("rejects malformed numeric usage consistently across resource, scope and span attributes", async () => {
for (const [key, value] of [
["gen_ai.usage.input_cost", "-1"],
["gen_ai.usage.input_cost", "Infinity"],
["gen_ai.usage.input_tokens", "-1"],
["gen_ai.usage.output_tokens", "1.5"],
["gen_ai.usage.total_tokens", "2147483648"],
]) {
for (const level of ["resource", "scope", "span"]) {
const { app, calls, usagePayload } = createWebhookTestServer()
const payload = usagePayload({ requestId: "invalid-numeric", eventId: "invalid-numeric", generationId: "invalid-numeric", requestModel: "z-ai/glm-5.2", responseModel: "z-ai/glm-5.2" })
const attributes = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.attributes.filter((attr) => attr.key !== key)
attributes.push(attribute(key!, value!))
const response = await app.fetch(webhookRequest({ resourceSpans: [{
resource: { attributes: level === "resource" ? attributes : [] },
scopeSpans: [{ scope: { attributes: level === "scope" ? attributes : [] }, spans: [{ startTimeUnixNano: "1700000000000000000", attributes: level === "span" ? attributes : [] }] }],
}] }))
assert.equal(response.status, 400, `${level} ${key}=${value}`)
assert.deepEqual(await responseJson(response), { ok: false, ingested: 0, skipped: 0, deferred: 0, invalid: 1, failed: 0 })
assert.equal(calls.settleUsage, 0)
}
}
})