Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
// Strict telemetry schemas must reject raw errors and unknown enums while round-tripping valid payloads without coercion.
import { describe, expect, it } from 'vitest'
import { eventSchemas } from './telemetry-events'
describe('feature_interaction_usage_bucket_reached schema', () => {
it('accepts a valid bucket payload', () => {
const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
feature_id: 'browser-tab-created',
feature_category: 'browser',
count_bucket: 'count_3_4',
bucket_source: 'crossed_now',
nth_repo_added: 2
})
expect(parsed.success).toBe(true)
})
it('rejects unknown enum values and mismatched categories', () => {
const valid = {
feature_id: 'github-tasks',
feature_category: 'task_management',
count_bucket: 'count_1',
bucket_source: 'observed_existing'
}
expect(
eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
...valid,
feature_id: 'unknown-feature'
}).success
).toBe(false)
expect(
eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
...valid,
feature_category: 'browser'
}).success
).toBe(false)
expect(
eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
...valid,
count_bucket: 'count_4'
}).success
).toBe(false)
expect(
eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
...valid,
bucket_source: 'renderer'
}).success
).toBe(false)
})
it('rejects raw privacy fields via .strict()', () => {
const rawFields = [
'prompt',
'command',
'path',
'repo',
'branch',
'url',
'hostname',
'error',
'text',
'query',
'result_label',
'workspace_name',
'setting_name',
'target_id',
'annotation_text',
'dom_snippet',
'screenshot',
'page_title',
'trusted_directory',
'trigger_x',
'trigger_y',
'focus_state',
'minimize_state',
'audio',
'transcript',
'model',
'device',
'error_detail'
]
for (const field of rawFields) {
const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
feature_id: 'browser-annotations-sent-to-agent',
feature_category: 'browser',
count_bucket: 'count_1',
bucket_source: 'crossed_now',
[field]: 'raw'
})
expect(parsed.success).toBe(false)
}
})
})
describe('app_starred_orca schema', () => {
it('accepts cohort context on successful app star telemetry', () => {
const parsed = eventSchemas.app_starred_orca.safeParse({
source: 'settings',
nth_repo_added: 2
})
expect(parsed.success).toBe(true)
})
it('rejects unknown app star source values', () => {
const parsed = eventSchemas.app_starred_orca.safeParse({
source: 'github_website'
})
expect(parsed.success).toBe(false)
})
it('rejects extra keys via .strict()', () => {
const parsed = eventSchemas.app_starred_orca.safeParse({
source: 'landing',
repo: 'stablyai/orca'
})
expect(parsed.success).toBe(false)
})
})
describe('star_nag_outcome schema', () => {
const valid = {
outcome: 'shown',
source: 'threshold',
mode: 'gh',
threshold: 35,
agents_since_baseline: 35,
agents_since_baseline_bucket: '35-69',
nth_repo_added: 2
}
it('accepts a strict valid payload with cohort context', () => {
expect(eventSchemas.star_nag_outcome.safeParse(valid).success).toBe(true)
})
it('accepts next_threshold only as a positive integer for deferrals', () => {
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
outcome: 'dismissed',
next_threshold: 70
}).success
).toBe(true)
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
outcome: 'later',
next_threshold: 70
}).success
).toBe(true)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 0 }).success).toBe(
false
)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 1.5 }).success).toBe(
false
)
expect(
eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'shown', next_threshold: 70 })
.success
).toBe(false)
})
it('accepts cooldown_days only for deferrals', () => {
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
outcome: 'later',
cooldown_days: 30
}).success
).toBe(true)
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
outcome: 'dismissed',
cooldown_days: 30
}).success
).toBe(true)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, cooldown_days: 0 }).success).toBe(
false
)
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
outcome: 'opened_repo',
cooldown_days: 30
}).success
).toBe(false)
})
it('accepts new star nag action outcomes', () => {
for (const outcome of [
'star_clicked',
'direct_star_succeeded',
'direct_star_failed',
'opened_repo',
'later'
]) {
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome }).success).toBe(true)
}
})
it('rejects unknown outcome source mode and bucket values', () => {
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'ignored' }).success).toBe(
false
)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, source: 'renderer' }).success).toBe(
false
)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, mode: 'desktop' }).success).toBe(
false
)
expect(
eventSchemas.star_nag_outcome.safeParse({
...valid,
agents_since_baseline_bucket: '35+'
}).success
).toBe(false)
})
it('rejects malformed numeric fields and raw extra fields', () => {
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: -1 }).success).toBe(false)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: 1.5 }).success).toBe(
false
)
expect(
eventSchemas.star_nag_outcome.safeParse({ ...valid, agents_since_baseline: -1 }).success
).toBe(false)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, nth_repo_added: -1 }).success).toBe(
false
)
expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, error: 'gh failed' }).success).toBe(
false
)
expect(
eventSchemas.star_nag_outcome.safeParse({ ...valid, url: 'https://github.com' }).success
).toBe(false)
})
})
describe('agent_error schema', () => {
it('round-trips a minimal {error_class, agent_kind} payload', () => {
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'unknown',
agent_kind: 'claude-code'
})
expect(parsed.success).toBe(true)
})
// Privacy boundary: raw errors must never enter analytics.
it('rejects error_message via .strict()', () => {
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'unknown',
agent_kind: 'claude-code',
error_message: 'boom at /Users/alice/secret/path'
})
expect(parsed.success).toBe(false)
})
it('rejects error_stack via .strict()', () => {
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'unknown',
agent_kind: 'claude-code',
error_stack: 'Error: boom\n at /Users/alice/...'
})
expect(parsed.success).toBe(false)
})
it('rejects error_name (deferred — schema is enum-only)', () => {
// An earlier draft allowed error_name; reintroducing it requires revisiting this privacy boundary.
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'unknown',
agent_kind: 'claude-code',
error_name: 'BinaryNotFound'
})
expect(parsed.success).toBe(false)
})
it('rejects unknown error_class enum values', () => {
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'made_up_class',
agent_kind: 'claude-code'
})
expect(parsed.success).toBe(false)
})
it('rejects unknown agent_kind enum values', () => {
const parsed = eventSchemas.agent_error.safeParse({
error_class: 'unknown',
agent_kind: 'made_up_agent'
})
expect(parsed.success).toBe(false)
})
})
describe('daemon_lifecycle schema', () => {
it('round-trips a startup replace payload', () => {
const parsed = eventSchemas.daemon_lifecycle.safeParse({
transition: 'replaced',
reason: 'stale_bundle',
live_session_count_bucket: '0'
})
expect(parsed.success).toBe(true)
})
it('round-trips a retirement payload', () => {
const parsed = eventSchemas.daemon_lifecycle.safeParse({
transition: 'retired',
reason: 'died_respawn',
live_session_count_bucket: 'unknown'
})
expect(parsed.success).toBe(true)
})
// Privacy boundary: both union members must reject raw paths/versions/exact counts (STA-2376).
it('rejects raw paths, versions, and unbucketed counts via .strict()', () => {
const bases = [
{ transition: 'replaced', reason: 'failed_health_check', live_session_count_bucket: '2-5' },
{ transition: 'retired', reason: 'died_respawn', live_session_count_bucket: 'unknown' }
]
for (const base of bases) {
for (const leak of [
{ daemon_path: '/Users/alice/Orca.app' },
{ daemon_app_version: '1.4.129' },
{ live_session_count: 3 }
]) {
const parsed = eventSchemas.daemon_lifecycle.safeParse({ ...base, ...leak })
expect(parsed.success).toBe(false)
}
// Validate the base so unrelated schema errors cannot mask a privacy leak.
expect(eventSchemas.daemon_lifecycle.safeParse(base).success).toBe(true)
}
})
it('rejects unknown reason and bucket enum values', () => {
expect(
eventSchemas.daemon_lifecycle.safeParse({
transition: 'replaced',
reason: 'made_up_reason',
live_session_count_bucket: '0'
}).success
).toBe(false)
expect(
eventSchemas.daemon_lifecycle.safeParse({
transition: 'replaced',
reason: 'stale_bundle',
live_session_count_bucket: '99'
}).success
).toBe(false)
})
it('rejects reasons and fields that do not belong to the transition', () => {
expect(
eventSchemas.daemon_lifecycle.safeParse({
transition: 'replaced',
reason: 'died_respawn',
live_session_count_bucket: 'unknown'
}).success
).toBe(false)
expect(
eventSchemas.daemon_lifecycle.safeParse({
transition: 'retired',
reason: 'failed_health_check',
live_session_count_bucket: 'unknown'
}).success
).toBe(false)
})
})
describe('workspace_created schema', () => {
it('rejects unknown source', () => {
const parsed = eventSchemas.workspace_created.safeParse({
source: 'carrier_pigeon',
from_existing_branch: false
})
expect(parsed.success).toBe(false)
})
it('accepts a valid payload', () => {
const parsed = eventSchemas.workspace_created.safeParse({
source: 'command_palette',
from_existing_branch: true
})
expect(parsed.success).toBe(true)
})
})
describe('agent_started schema', () => {
it('requires all three keys', () => {
const parsed = eventSchemas.agent_started.safeParse({
agent_kind: 'claude-code',
launch_source: 'sidebar'
})
expect(parsed.success).toBe(false)
})
})
describe('agent_prompt_sent schema', () => {
it('accepts a hook-confirmed prompt-send payload with cohort context', () => {
const parsed = eventSchemas.agent_prompt_sent.safeParse({
agent_kind: 'codex',
launch_source: 'unknown',
request_kind: 'followup',
nth_repo_added: 1
})
expect(parsed.success).toBe(true)
})
it('rejects prompt text via .strict()', () => {
const parsed = eventSchemas.agent_prompt_sent.safeParse({
agent_kind: 'claude-code',
launch_source: 'unknown',
request_kind: 'followup',
prompt: 'please inspect /Users/alice/private-repo'
})
expect(parsed.success).toBe(false)
})
})
describe('agent_hook_install_failed schema', () => {
it('accepts only the fixed error categories', () => {
for (const error_message of ['EACCES', 'E2BIG', 'unknown']) {
expect(
eventSchemas.agent_hook_install_failed.safeParse({ agent: 'codex', error_message }).success
).toBe(true)
}
})
it('rejects raw error messages and path-bearing codes (#21492)', () => {
for (const error_message of [
"EACCES: permission denied, open '/home/someone/.codex/config.toml'",
'EACCES /home/someone',
`E${'A'.repeat(16)}`,
'Timed out waiting for another managed-hook install to finish'
]) {
expect(
eventSchemas.agent_hook_install_failed.safeParse({ agent: 'codex', error_message }).success
).toBe(false)
}
})
})
describe('agent_hook_unattributed schema', () => {
it('accepts the two bounded attribution failure reasons', () => {
for (const reason of ['empty_pane_key', 'unknown_tab_id'] as const) {
expect(eventSchemas.agent_hook_unattributed.safeParse({ reason }).success).toBe(true)
}
})
it('rejects extra payload fields via .strict()', () => {
const parsed = eventSchemas.agent_hook_unattributed.safeParse({
reason: 'unknown_tab_id',
pane_key: 'tab-secret:1'
})
expect(parsed.success).toBe(false)
})
})
describe('add_repo_setup_step_action schema', () => {
it('rejects unknown action enum values', () => {
const parsed = eventSchemas.add_repo_setup_step_action.safeParse({
action: 'export_to_pdf'
})
expect(parsed.success).toBe(false)
})
it('rejects extra keys via .strict()', () => {
const parsed = eventSchemas.add_repo_setup_step_action.safeParse({
action: 'skip',
repo_name: 'orca' // raw repo names are UGC — must not cross the wire
})
expect(parsed.success).toBe(false)
})
})
describe('add_repo_default_checkout_handoff schema', () => {
it('accepts bounded handoff outcome enums', () => {
const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({
source: 'clone_url',
result: 'opened_default_checkout',
reason: 'detected_default_checkout',
nth_repo_added: 1
})
expect(parsed.success).toBe(true)
})
it('rejects raw repo/path context via .strict()', () => {
const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({
source: 'local_folder_picker',
result: 'revealed_project',
reason: 'no_default_checkout',
repo_name: 'secret-repo',
path: '/Users/alice/secret-repo'
})
expect(parsed.success).toBe(false)
})
})
describe('workspace_create_failed schema', () => {
it('rejects unknown error_class values', () => {
const parsed = eventSchemas.workspace_create_failed.safeParse({
source: 'sidebar',
error_class: 'cosmic_ray'
})
expect(parsed.success).toBe(false)
})
// Privacy boundary: raw workspace-create errors must never enter analytics.
it('rejects error_message via .strict()', () => {
const parsed = eventSchemas.workspace_create_failed.safeParse({
source: 'sidebar',
error_class: 'git_failed',
error_message: 'fatal: cannot create work tree at /Users/alice/secret'
})
expect(parsed.success).toBe(false)
})
it('rejects error_stack via .strict()', () => {
const parsed = eventSchemas.workspace_create_failed.safeParse({
source: 'sidebar',
error_class: 'git_failed',
error_stack: 'Error: cannot create work tree\n at /Users/alice/...'
})
expect(parsed.success).toBe(false)
})
})
describe('settings_changed schema', () => {
it('accepts the Chat UI switch as a boolean adoption signal', () => {
expect(
eventSchemas.settings_changed.safeParse({
setting_key: 'experimentalNativeChat',
value_kind: 'bool'
}).success
).toBe(true)
})
it('accepts the browser extensions switch as a boolean adoption signal', () => {
expect(
eventSchemas.settings_changed.safeParse({
setting_key: 'experimentalBrowserExtensions',
value_kind: 'bool'
}).success
).toBe(true)
})
it.each(['experimentalStructuredNativeChat', 'openAgentTabsInChatByDefault'])(
'rejects the retired %s key',
(settingKey) => {
expect(
eventSchemas.settings_changed.safeParse({ setting_key: settingKey, value_kind: 'bool' })
.success
).toBe(false)
}
)
it('rejects non-whitelisted setting keys', () => {
const parsed = eventSchemas.settings_changed.safeParse({
setting_key: 'telemetryOptIn', // deliberately excluded from the whitelist
value_kind: 'bool'
})
expect(parsed.success).toBe(false)
})
})
describe('remote_outbound_budget_close schema', () => {
it('round-trips every emitter', () => {
for (const emitter of ['size', 'queue']) {
expect(eventSchemas.remote_outbound_budget_close.safeParse({ emitter }).success).toBe(true)
}
})
it('rejects an unknown emitter and any payload-describing extra key', () => {
expect(eventSchemas.remote_outbound_budget_close.safeParse({ emitter: 'other' }).success).toBe(
false
)
expect(
eventSchemas.remote_outbound_budget_close.safeParse({
emitter: 'size',
byte_length: 4194305
}).success
).toBe(false)
})
})