A desktop coding agent with free access to frontier models.
TypeScript
import { expect } from "vitest";
import { denFetch, provisionOrg } from "@omnirush/behaviors";
import type { DenSession } from "@omnirush/behaviors";
import { spec } from "@omnirush/testkit";
import { orgModelAnalyticsWorld } from "../worlds/org-model-analytics.ts";
const test = spec.world(orgModelAnalyticsWorld, { timeout: 900_000, needs: {} });
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`);
return value;
}
function auth(session: DenSession, orgId: string): Record<string, string> {
return {
authorization: `Bearer ${session.token}`,
"x-omnirush-org-id": orgId,
};
}
async function organizationIdByName(session: DenSession, name: string): Promise<string> {
const result = await denFetch(session, "/v1/me/orgs", {
headers: { authorization: `Bearer ${session.token}` },
signal: AbortSignal.timeout(10_000),
});
const organizations = isRecord(result.body) && Array.isArray(result.body.orgs)
? result.body.orgs.filter(isRecord)
: [];
const organization = organizations.find((item) => item.name === name);
const orgId = organization && typeof organization.id === "string" ? organization.id : "";
if (!result.response.ok || !orgId) {
throw new Error(`Finding ${name} failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
}
return orgId;
}
function telemetryEvent(sessionId: string, type: "session.active" | "task.started", model: string, selection: "default" | "manual") {
return {
type,
timestamp: new Date().toISOString(),
source: "app",
sessionId,
dimensions: [
{ type: "model", value: model, label: model },
{ type: "model_selection", value: selection, label: selection },
],
};
}
async function readAnalytics(session: DenSession, orgId: string): Promise<Record<string, unknown>> {
const result = await denFetch(session, "/v1/telemetry/analytics", {
headers: auth(session, orgId),
signal: AbortSignal.timeout(10_000),
});
if (result.response.status === 402) {
throw new Error("Model analytics proof requires the local default DEN_PLAN_GATING_ENABLED=false; do not enable plan gating for this spec.");
}
if (!result.response.ok) {
throw new Error(`Reading model analytics failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
}
return requireRecord(requireRecord(result.body, "analytics response").models, "analytics models");
}
test("organization model analytics aggregate session dimensions without cross-org leakage", { timeout: 600_000 }, async ({ evidence, world, user }) => {
const { den } = world;
const primaryOrgId = await organizationIdByName(den.admin, "Analytics team");
const otherOrg = await provisionOrg(den.ref, {});
const events = [
telemetryEvent("model-analytics-session-a", "session.active", "prov/model-a", "default"),
telemetryEvent("model-analytics-session-a", "task.started", "prov/model-a", "default"),
telemetryEvent("model-analytics-session-b", "session.active", "prov/model-b", "manual"),
telemetryEvent("model-analytics-session-b", "task.started", "prov/model-b", "manual"),
];
const ingested = await denFetch(den.admin, "/v1/telemetry/ingest", {
method: "POST",
headers: auth(den.admin, primaryOrgId),
body: JSON.stringify({ events }),
signal: AbortSignal.timeout(10_000),
});
expect(ingested.response.status, `Telemetry ingestion: ${ingested.text.slice(0, 500)}`).toBe(204);
const primaryModels = await readAnalytics(den.admin, primaryOrgId);
const usage30d = Array.isArray(primaryModels.usage30d) ? primaryModels.usage30d : [];
expect(usage30d).toHaveLength(2);
expect(usage30d).toEqual(expect.arrayContaining([
{ id: "prov/model-a", label: "prov/model-a", sessions: 1 },
{ id: "prov/model-b", label: "prov/model-b", sessions: 1 },
]));
expect(primaryModels.selection30d).toEqual({ default: 1, manual: 1 });
evidence.recordAssertionEvidence(
"Model usage is aggregated by distinct session",
JSON.stringify(usage30d),
usage30d.length === 2,
);
evidence.recordAssertionEvidence(
"Default and manual model selections are counted",
JSON.stringify(primaryModels.selection30d),
isRecord(primaryModels.selection30d)
&& primaryModels.selection30d.default === 1
&& primaryModels.selection30d.manual === 1,
);
const otherModels = await readAnalytics(otherOrg.admin, otherOrg.orgId);
expect(otherModels.usage30d).toEqual([]);
expect(otherModels.selection30d).toEqual({ default: 0, manual: 0 });
evidence.recordAssertionEvidence(
"Model analytics are isolated by organization",
JSON.stringify(otherModels),
Array.isArray(otherModels.usage30d) && otherModels.usage30d.length === 0,
);
await user.see({ text: "Understand how your team works in OmniRush.ai" }, { timeoutMs: 60_000 });
await user.click({ role: "button", label: "Refresh analytics" });
await user.see({ text: /^prov\/model-a\s+1$/ }, { timeoutMs: 30_000 });
await user.see({ text: /^prov\/model-b\s+1$/ });
await user.notSee({ text: "No model usage yet" });
await user.screenshot();
evidence.recordAssertionEvidence("Usage and adoption shows the same model usage returned by the organization API", "Both model rows show exactly one session, matching the API totals, and the empty-state message is absent", true);
await world.analyticsStoreUnavailable(true);
const unavailable = await denFetch(den.admin, "/v1/telemetry/analytics", { headers: auth(den.admin, primaryOrgId) });
expect(unavailable.response.status).toBe(500);
await user.reload();
await user.see({ text: "Could not load analytics." }, { timeoutMs: 60_000 });
await user.notSee({ text: "No model usage yet" });
await user.notSee({ text: "No usage events yet" });
await world.analyticsStoreUnavailable(false);
await user.click({ role: "button", label: "Refresh analytics" });
await user.see({ text: "prov/model-a" }, { timeoutMs: 30_000 });
await user.notSee({ text: "Could not load analytics." });
evidence.recordAssertionEvidence("An analytics outage is shown as an error and can recover without inventing zero usage", "An analytics storage outage returned HTTP 500 and produced the error state with no empty charts; Refresh restored the previously ingested models after recovery", true);
await user.click({ role: "link", label: "Models & usage" });
await user.see({ text: "Understand your team’s OmniRush.ai Models activity, consumption, and shared limits." });
await user.notSee({ role: "button", label: "Subscribe" });
await user.click({ role: "link", label: /^Usage & adoption$/ });
await user.see({ text: "Understand how your team works in OmniRush.ai" });
await user.see({ text: "prov/model-b" });
evidence.recordAssertionEvidence("Shared navigation connects adoption and Models analytics", "Both views are reachable through the shared navigation and returning preserves access to the same organization’s model analytics", true);
});