Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
import { afterEach, describe, expect, it } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import proportions, { calculate as interval } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/proportions.mjs';
import adjust, { calculate as correction } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/adjust-pvalues.mjs';
import exact, { calculate as total } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/exact-total.mjs';
import { normalCritical } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/normal.mjs';
import { percentage, table } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/presentation.mjs';
import missing, { calculate as bounds } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/missing-binary-bounds.mjs';
import uncertainty, { calculate as uncertaintyGate } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/uncertainty-gate.mjs';
// Portfolio: valid / sparse experiments; reversed direction and wider levels;
// dependent or malformed data; complete-family error control; exact financial
// reconciliation; unchanged inputs and runner failure/recovery. Numerical
// anchors are standard-normal quantiles and statsmodels' published Fagerland
// table, not values generated by the helpers under test.
const tempRoots: string[] = [];
afterEach(() => { for (const root of tempRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
const binomial = (events: number, n: number, confidence_level = 0.95) => interval({ design: 'independent-binomial', groups: [{ events, total: n }], confidence_level });
describe('reusable analytical calculations', () => {
it('rejects generic confidence scores and selects evidence-appropriate uncertainty', () => {
expect(uncertaintyGate({ claim: 'causal-effect', evidence: 'unidentified-observational' })).toMatchObject({
generic_numeric_confidence_allowed: false,
source_defined_score_allowed: false,
result: 'do-not-report-numeric-confidence',
reporting_requirements: [
expect.stringContaining('Preserve source units'),
expect.stringContaining('unit and denominator'),
],
});
expect(uncertaintyGate({ claim: 'causal-effect', evidence: 'identified-design' }).recommendation).toContain('report its interval');
expect(uncertaintyGate({ claim: 'descriptive-estimate', evidence: 'sampling-model' }).recommendation).toContain('sampling interval');
expect(uncertaintyGate({ claim: 'deterministic-result', evidence: 'complete-scope' }).recommendation).toContain('reconciliation');
expect(uncertaintyGate({ claim: 'source-defined-score', evidence: 'source-defined-rule' })).toMatchObject({
generic_numeric_confidence_allowed: false,
source_defined_score_allowed: true,
result: 'compute-source-defined-rule',
});
});
it.each([
{},
{ claim: 'causal-effect', evidence: 'intuition' },
{ claim: 'belief', evidence: 'insufficient' },
{ claim: 'causal-effect', evidence: 'unidentified-observational', score: 0 },
])('rejects malformed confidence eligibility requests without returning a score: %j', async value => {
const result = await uncertainty({ args: ['--json', JSON.stringify(value)] });
expect(result).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
expect(result).not.toHaveProperty('generic_numeric_confidence_allowed');
});
it('bounds all eligible outcomes instead of silently treating missing as failure or dropping them', () => {
const groups = [{ label: 'A', successes: 3, failures: 1, missing: 4 }, { label: 'B', successes: 4, failures: 4, missing: 0 }];
const result = bounds({ groups });
expect(result.groups[0]).toMatchObject({ observed_rate: 0.75, total: 8, bounds: { low: 0.375, high: 0.875 } });
expect(result.difference.bounds).toEqual({ low: -0.125, high: 0.375 });
expect(result.report).toContain('| 37.5 to 87.5 | % |');
expect(result.report).toContain('| -12.5 to 37.5 | percentage points |');
expect(result).not.toHaveProperty('confidence_level');
expect(groups[0].missing).toBe(4);
});
it('matches exhaustive binary completions, including complete and entirely missing groups', () => {
for (let successes = 0; successes <= 3; successes++) for (let failures = 0; failures <= 3; failures++) for (let missing = 0; missing <= 3; missing++) {
if (!successes && !failures && !missing) continue;
const group = bounds({ groups: [{ successes, failures, missing }] }).groups[0];
const rates = Array.from({ length: 2 ** missing }, (_, mask) => {
const bits = mask.toString(2).split('').filter(bit => bit === '1').length;
return (successes + bits) / (successes + failures + missing);
});
expect(group.bounds).toEqual({ low: Math.min(...rates), high: Math.max(...rates) });
if (!successes && !failures) expect(group.observed_rate).toBeNull();
}
const groups = [{ successes: 1, failures: 3, missing: 2 }, { successes: 3, failures: 1, missing: 4 }];
const forward = bounds({ groups }).difference.bounds;
const reverse = bounds({ groups: [...groups].reverse() }).difference.bounds;
expect(reverse.low).toBeCloseTo(-forward.high, 14);
expect(reverse.high).toBeCloseTo(-forward.low, 14);
});
it.each([
{ groups: [{ successes: 0, failures: 0, missing: 0 }] },
{ groups: [{ successes: '2', failures: 1, missing: 1 }] },
{ groups: [{ successes: 2, failures: -1, missing: 1 }] },
{ groups: [{ successes: 2, failures: 1 }] },
{ groups: [{ successes: Number.MAX_SAFE_INTEGER, failures: 1, missing: 0 }] },
{ groups: [{ successes: 2, failures: 1, missing: 1 }], confidence_level: 0.95 },
])('rejects inconsistent or sampling-interval requests without fabricated bounds: %j', async value => {
const result = await missing({ args: ['--json', JSON.stringify(value)] });
expect(result).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
expect(result).not.toHaveProperty('groups');
});
it('delivers computed percentages and percentage-point differences with denominators, without changing raw precision', () => {
const result = interval({ design: 'independent-binomial', groups: [
{ label: 'Treatment', events: 600, total: 5000 },
{ label: 'Control', events: 500, total: 5000 },
] });
expect(result.groups[0].rate).toBe(0.12);
expect(result.difference.estimate).toBeCloseTo(0.02, 14);
expect(result.report).toContain('| Treatment | 12 | % | 600 / 5000 |');
expect(result.report).toContain('| Control | 10 | % | 500 / 5000 |');
expect(result.report).toContain('| 2 | percentage points | 5000 vs 5000 independent units |');
expect(result.report).toContain(`${percentage(result.difference.interval.low)} to ${percentage(result.difference.interval.high)}`);
expect(result.report).toContain('95% confidence intervals (not effect probabilities)');
expect(result.report).toContain('not verified by this calculation');
});
it('keeps tiny nonzero percentages visible and normalizes exact zero', () => {
expect(percentage(0)).toBe('0');
expect(percentage(-0)).toBe('0');
expect(percentage(1)).toBe('100');
expect(percentage(-0.02)).toBe('-2');
expect(percentage(0.000000001)).toBe('1.000e-7');
expect(percentage(-0.000000001)).toBe('-1.000e-7');
const sparse = binomial(1, 1000000000);
expect(sparse.report).toContain('| 1.000e-7 | % | 1 / 1000000000 |');
expect(sparse.groups[0].rate).toBe(1e-9);
});
it('does not round a below-one confidence level or event rate into 100 percent certainty', () => {
for (const level of [0.9999999, 1 - Number.EPSILON]) {
const result = binomial(5, 10, level);
const shown = Number(result.report.match(/approximate (\S+)% confidence/u)[1]);
expect(shown).toBeLessThan(100);
expect(shown).toBeGreaterThan(99.9999);
expect(result.confidence_level).toBe(level);
}
expect(Number(percentage(0.9999999))).toBeLessThan(100);
expect(Number(percentage(-0.9999999))).toBeGreaterThan(-100);
});
it('preserves exact money strings and units in every reported reconciliation row', () => {
const result = total({ values: ['9007199254740993.10', '0.20', '-0.05'], unit: 'USD', expected: '9007199254740993.26' });
expect(result.report).toContain('| Computed total | 9007199254740993.25 | USD |');
expect(result.report).toContain('| Expected total | 9007199254740993.26 | USD |');
expect(result.report).toContain('| Computed minus expected | -0.01 | USD |');
expect(result.report).toContain('Summed 3 supplied values; no rounding');
expect(result.report).toContain('not source completeness or authenticity');
expect(total({ values: ['-0.1', '0.1'], unit: 'count' }).report).toContain('| 0.0 | count |');
});
it('escapes table-breaking source labels without altering machine-readable labels', () => {
const label = 'A|B\n[click](x)<tag>_bold_';
const result = interval({ design: 'independent-binomial', groups: [{ label, events: 1, total: 10 }] });
expect(result.groups[0].label).toBe(label);
expect(result.report).toContain('A\\|B \\[click\\](x)\\<tag\\>\\_bold\\_');
expect(table(['Metric'], [['x\r\ny']])).toContain('| x y |');
expect(table(['Metric'], [['x\ry']])).toContain('| x y |');
});
it('bounds large-family display explicitly while retaining every hypothesis and original ordering', () => {
const pvalues = Array.from({ length: 21 }, (_, i) => (i + 1) / 100);
const result = correction({ pvalues, method: 'holm' });
expect(result.report).toContain('family-wise error rate; alpha=0.05');
expect(result.report).toContain('Showing 20 of 21 rows');
expect(result.report).toContain('the JSON arrays contain the complete results');
expect(result.report).toContain('| 19 | 0.2 |');
expect(result.report).not.toContain('| 20 | 0.21 |');
expect(result.adjusted_pvalues).toHaveLength(21);
expect(result.pvalues).toEqual(pvalues);
expect(result.reject).toHaveLength(21);
});
it('uses accurate normal critical values rather than treating confidence level as an output score', () => {
for (const [level, z] of [[0.90, 1.6448536269514722], [0.95, 1.959963984540054], [0.99, 2.5758293035489004]]) {
expect(normalCritical(level)).toBeCloseTo(z, 12);
}
});
it('retains sampling uncertainty with zero or all conversions', () => {
const zero = binomial(0, 10).groups[0];
expect(zero.rate).toBe(0);
expect(zero.interval.low).toBe(0);
expect(zero.interval.high).toBeCloseTo(0.2775327998628892, 12);
const all = binomial(10, 10).groups[0];
expect(all.interval.low).toBeCloseTo(0.7224672001371107, 12);
expect(all.interval.high).toBe(1);
});
it('matches the independently published Newcombe table and preserves difference direction', () => {
// statsmodels/stats/tests/test_proportion.py::test_confint_2indep:
// Fagerland et al. (2015), 7/34 minus 1/34, CI [0.019, 0.340], atol .005.
const groups = [{ events: 7, total: 34 }, { events: 1, total: 34 }];
const result = interval({ design: 'independent-binomial', groups });
expect(result.difference.estimate).toBeCloseTo(6 / 34, 14);
expect(Math.abs(result.difference.interval.low - 0.019)).toBeLessThan(0.005);
expect(Math.abs(result.difference.interval.high - 0.340)).toBeLessThan(0.005);
const reverse = interval({ design: 'independent-binomial', groups: [...groups].reverse() });
expect(reverse.difference.estimate).toBeCloseTo(-result.difference.estimate, 14);
expect(reverse.difference.interval.low).toBeCloseTo(-result.difference.interval.high, 14);
expect(reverse.difference.interval.high).toBeCloseTo(-result.difference.interval.low, 14);
});
it('widens intervals as requested coverage increases across sparse and large samples', () => {
for (const n of [1, 7, 100, 1000000]) for (const events of [0, 1, Math.floor(n / 2), n]) {
const a = binomial(events, n, 0.90).groups[0].interval;
const b = binomial(events, n, 0.99).groups[0].interval;
expect(b.low).toBeLessThanOrEqual(a.low + 1e-14);
expect(b.high).toBeGreaterThanOrEqual(a.high - 1e-14);
expect(b.low).toBeGreaterThanOrEqual(0);
expect(b.high).toBeLessThanOrEqual(1);
expect(b.high).toBeGreaterThan(b.low);
}
});
it.each([
{ design: 'paired', groups: [{ events: 2, total: 10 }] },
{ design: 'independent-binomial', groups: [{ events: 2, total: 0 }] },
{ design: 'independent-binomial', groups: [{ events: 11, total: 10 }] },
{ design: 'independent-binomial', groups: [{ events: '2', total: 10 }] },
{ design: 'independent-binomial', groups: [{ events: 1.5, total: 10 }] },
{ design: 'independent-binomial', groups: [{ events: 2, total: 10 }], confidence_level: null },
{ design: 'independent-binomial', groups: [{ events: 2, total: 10 }], confidence_level: 1 },
{ design: 'independent-binomial', groups: [{ events: 2, total: 10, weights: [1] }] },
])('returns no estimate for unsupported or invalid analytical inputs: %j', async value => {
const result = await proportions({ args: ['--json', JSON.stringify(value)] });
expect(result.ok).toBe(false);
expect(result.error.code).toBe('INVALID_INPUT');
expect(result.error.message.length).toBeGreaterThan(10);
expect(result).not.toHaveProperty('groups');
});
it('distinguishes family-wise and discovery-rate decisions and retains hypothesis order', () => {
const pvalues = [0.04, 0.001, 0.03, 0.02];
const holm = correction({ pvalues, method: 'holm' });
expect(holm.adjusted_pvalues).toEqual([0.06, 0.004, 0.06, 0.06]);
expect(holm.reject).toEqual([false, true, false, false]);
const bh = correction({ pvalues, method: 'bh' });
expect(bh.adjusted_pvalues).toEqual([0.04, 0.004, 0.04, 0.04]);
expect(bh.reject).toEqual([true, true, true, true]);
const by = correction({ pvalues, method: 'by' });
expect(by.adjusted_pvalues[0]).toBeCloseTo(1 / 12, 14);
expect(by.adjusted_pvalues[1]).toBeCloseTo(1 / 120, 14);
expect(by.reject).toEqual([false, true, false, false]);
expect(pvalues).toEqual([0.04, 0.001, 0.03, 0.02]);
});
it('correction handles ties, boundaries and permutations without reducing raw p-values', () => {
const pvalues = [1, 0.05, 0, 0.05, 0.4];
for (const method of ['holm', 'bh', 'by']) {
const result = correction({ pvalues, method });
const reverse = correction({ pvalues: [...pvalues].reverse(), method });
expect(reverse.adjusted_pvalues.reverse()).toEqual(result.adjusted_pvalues);
expect(result.adjusted_pvalues[1]).toBe(result.adjusted_pvalues[3]);
result.adjusted_pvalues.forEach((p: number, i: number) => {
expect(p).toBeGreaterThanOrEqual(pvalues[i]);
expect(p).toBeLessThanOrEqual(1);
});
}
});
it.each([{ pvalues: [], method: 'bh' }, { pvalues: [null], method: 'holm' }, { pvalues: [1.1], method: 'bh' }, { pvalues: [0.1], method: 'auto' }, { pvalues: [0.1], method: 'holm', alpha: null }])('rejects invalid test families without inventing corrected results: %j', async value => {
const result = await adjust({ args: ['--json', JSON.stringify(value)] });
expect(result.ok).toBe(false);
expect(result).not.toHaveProperty('adjusted_pvalues');
});
it('reconciles exact monetary totals beyond the safe-integer boundary, including refunds', () => {
const value = { values: ['9007199254740993.10', '0.20', '-0.05'], unit: 'source cents', expected: '9007199254740993.26' };
const result = total(value);
expect(result).toMatchObject({ count: 3, total: '9007199254740993.25', unit: 'source cents', difference: '-0.01', matches: false });
expect(total({ ...value, expected: result.total }).matches).toBe(true);
const partition = total({ values: value.values.slice(0, 2), unit: value.unit });
expect(total({ values: [partition.total, value.values[2]], unit: value.unit }).total).toBe(result.total);
expect(value.values).toEqual(['9007199254740993.10', '0.20', '-0.05']);
});
it('handles empty input and signed zero without claiming input completeness', () => {
expect(total({ values: [], unit: 'count' })).toMatchObject({ count: 0, total: '0' });
expect(total({ values: ['-0.02', '0.020'], unit: 'unknown currency cents' }).total).toBe('0.000');
});
it.each([['1e3'], ['1,000'], [0.1], [null], ['2元']])('rejects lossy or ambiguous money input: %j', async values => {
const result = await exact({ args: ['--json', JSON.stringify({ values, unit: 'cents' })] });
expect(result.ok).toBe(false);
expect(result.error.message).toContain('decimal strings');
expect(result).not.toHaveProperty('total');
});
it('reads a BOM/Unicode-spaced input through the real runner and reports recoverable failures without writes', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'analysis calculations '));
tempRoots.push(root);
const source = path.join(root, '输入 amounts.json');
const body = '\uFEFF' + JSON.stringify({ values: ['0.1', '0.2'], unit: 'cents', expected: '0.3' });
fs.writeFileSync(source, body);
const runner = (args: string[]) => spawnSync(process.execPath, [path.resolve('bin/run-skill.cjs'), 'data-analysis', 'exact-total', '--', ...args], {
cwd: root, encoding: 'utf8', timeout: 30000,
env: { ...process.env, ELECTRON_RUN_AS_NODE: '1', ORKAS_PC_DIR: process.cwd(), ORKAS_WORKSPACE_ROOT: path.join(root, 'data'), ORKAS_RUN_SKILL_DIR: path.resolve('resources/builtin/marketplace/skills/08837be6032e') },
});
const success = runner(['--input', source]);
expect(success.status, success.stdout + success.stderr).toBe(0);
expect(success.stderr).toBe('');
expect(JSON.parse(success.stdout)).toMatchObject({ ok: true, total: '0.3', matches: true });
const bad = runner(['--json', '{"values":[0.1],"unit":"cents"}']);
expect(bad.status).not.toBe(0);
expect(bad.stderr).toBe('');
expect(JSON.parse(bad.stdout)).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
const missing = await exact({ args: ['--input', path.join(root, 'absent.json')] });
expect(missing.error.message).toContain('Check the supplied path');
expect(missing.error.message).not.toContain(root);
expect(fs.readFileSync(source, 'utf8')).toBe(body);
expect(fs.readdirSync(root)).toEqual(['输入 amounts.json']);
});
it('rejects oversized, malformed and unknown-option input rather than computing a partial or differently configured result', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'analysis-input-limit-'));
tempRoots.push(root);
const source = path.join(root, 'oversized.json');
fs.writeFileSync(source, ' '.repeat(8 * 1024 * 1024 + 1));
for (const args of [
['--input', source],
['--json', '{'],
['--json', '{}', '--confidence', '0.99'],
['--json', '{"values":["1"],"unit":"cents","rounding":0}'],
]) {
const result = await exact({ args });
expect(result.ok).toBe(false);
expect(result.error.message.length).toBeGreaterThan(10);
expect(result).not.toHaveProperty('total');
}
expect(fs.statSync(source).size).toBe(8 * 1024 * 1024 + 1);
});
});