Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

19.1 KB
Raw Blame
import { afterEach, describe, expect, it } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import proportions, { calculate as interval } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/proportions.mjs';
import adjust, { calculate as correction } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/adjust-pvalues.mjs';
import exact, { calculate as total } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/exact-total.mjs';
import { normalCritical } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/normal.mjs';
import { percentage, table } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/presentation.mjs';
import missing, { calculate as bounds } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/missing-binary-bounds.mjs';
import uncertainty, { calculate as uncertaintyGate } from '../../resources/builtin/marketplace/skills/08837be6032e/scripts/uncertainty-gate.mjs';

// Portfolio: valid / sparse experiments; reversed direction and wider levels;
// dependent or malformed data; complete-family error control; exact financial
// reconciliation; unchanged inputs and runner failure/recovery. Numerical
// anchors are standard-normal quantiles and statsmodels' published Fagerland
// table, not values generated by the helpers under test.
const tempRoots: string[] = [];
afterEach(() => { for (const root of tempRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
const binomial = (events: number, n: number, confidence_level = 0.95) => interval({ design: 'independent-binomial', groups: [{ events, total: n }], confidence_level });

describe('reusable analytical calculations', () => {
  it('rejects generic confidence scores and selects evidence-appropriate uncertainty', () => {
    expect(uncertaintyGate({ claim: 'causal-effect', evidence: 'unidentified-observational' })).toMatchObject({
      generic_numeric_confidence_allowed: false,
      source_defined_score_allowed: false,
      result: 'do-not-report-numeric-confidence',
      reporting_requirements: [
        expect.stringContaining('Preserve source units'),
        expect.stringContaining('unit and denominator'),
      ],
    });
    expect(uncertaintyGate({ claim: 'causal-effect', evidence: 'identified-design' }).recommendation).toContain('report its interval');
    expect(uncertaintyGate({ claim: 'descriptive-estimate', evidence: 'sampling-model' }).recommendation).toContain('sampling interval');
    expect(uncertaintyGate({ claim: 'deterministic-result', evidence: 'complete-scope' }).recommendation).toContain('reconciliation');
    expect(uncertaintyGate({ claim: 'source-defined-score', evidence: 'source-defined-rule' })).toMatchObject({
      generic_numeric_confidence_allowed: false,
      source_defined_score_allowed: true,
      result: 'compute-source-defined-rule',
    });
  });

  it.each([
    {},
    { claim: 'causal-effect', evidence: 'intuition' },
    { claim: 'belief', evidence: 'insufficient' },
    { claim: 'causal-effect', evidence: 'unidentified-observational', score: 0 },
  ])('rejects malformed confidence eligibility requests without returning a score: %j', async value => {
    const result = await uncertainty({ args: ['--json', JSON.stringify(value)] });
    expect(result).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
    expect(result).not.toHaveProperty('generic_numeric_confidence_allowed');
  });

  it('bounds all eligible outcomes instead of silently treating missing as failure or dropping them', () => {
    const groups = [{ label: 'A', successes: 3, failures: 1, missing: 4 }, { label: 'B', successes: 4, failures: 4, missing: 0 }];
    const result = bounds({ groups });
    expect(result.groups[0]).toMatchObject({ observed_rate: 0.75, total: 8, bounds: { low: 0.375, high: 0.875 } });
    expect(result.difference.bounds).toEqual({ low: -0.125, high: 0.375 });
    expect(result.report).toContain('| 37.5 to 87.5 | % |');
    expect(result.report).toContain('| -12.5 to 37.5 | percentage points |');
    expect(result).not.toHaveProperty('confidence_level');
    expect(groups[0].missing).toBe(4);
  });

  it('matches exhaustive binary completions, including complete and entirely missing groups', () => {
    for (let successes = 0; successes <= 3; successes++) for (let failures = 0; failures <= 3; failures++) for (let missing = 0; missing <= 3; missing++) {
      if (!successes && !failures && !missing) continue;
      const group = bounds({ groups: [{ successes, failures, missing }] }).groups[0];
      const rates = Array.from({ length: 2 ** missing }, (_, mask) => {
        const bits = mask.toString(2).split('').filter(bit => bit === '1').length;
        return (successes + bits) / (successes + failures + missing);
      });
      expect(group.bounds).toEqual({ low: Math.min(...rates), high: Math.max(...rates) });
      if (!successes && !failures) expect(group.observed_rate).toBeNull();
    }
    const groups = [{ successes: 1, failures: 3, missing: 2 }, { successes: 3, failures: 1, missing: 4 }];
    const forward = bounds({ groups }).difference.bounds;
    const reverse = bounds({ groups: [...groups].reverse() }).difference.bounds;
    expect(reverse.low).toBeCloseTo(-forward.high, 14);
    expect(reverse.high).toBeCloseTo(-forward.low, 14);
  });

  it.each([
    { groups: [{ successes: 0, failures: 0, missing: 0 }] },
    { groups: [{ successes: '2', failures: 1, missing: 1 }] },
    { groups: [{ successes: 2, failures: -1, missing: 1 }] },
    { groups: [{ successes: 2, failures: 1 }] },
    { groups: [{ successes: Number.MAX_SAFE_INTEGER, failures: 1, missing: 0 }] },
    { groups: [{ successes: 2, failures: 1, missing: 1 }], confidence_level: 0.95 },
  ])('rejects inconsistent or sampling-interval requests without fabricated bounds: %j', async value => {
    const result = await missing({ args: ['--json', JSON.stringify(value)] });
    expect(result).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
    expect(result).not.toHaveProperty('groups');
  });

  it('delivers computed percentages and percentage-point differences with denominators, without changing raw precision', () => {
    const result = interval({ design: 'independent-binomial', groups: [
      { label: 'Treatment', events: 600, total: 5000 },
      { label: 'Control', events: 500, total: 5000 },
    ] });
    expect(result.groups[0].rate).toBe(0.12);
    expect(result.difference.estimate).toBeCloseTo(0.02, 14);
    expect(result.report).toContain('| Treatment | 12 | % | 600 / 5000 |');
    expect(result.report).toContain('| Control | 10 | % | 500 / 5000 |');
    expect(result.report).toContain('| 2 | percentage points | 5000 vs 5000 independent units |');
    expect(result.report).toContain(`${percentage(result.difference.interval.low)} to ${percentage(result.difference.interval.high)}`);
    expect(result.report).toContain('95% confidence intervals (not effect probabilities)');
    expect(result.report).toContain('not verified by this calculation');
  });

  it('keeps tiny nonzero percentages visible and normalizes exact zero', () => {
    expect(percentage(0)).toBe('0');
    expect(percentage(-0)).toBe('0');
    expect(percentage(1)).toBe('100');
    expect(percentage(-0.02)).toBe('-2');
    expect(percentage(0.000000001)).toBe('1.000e-7');
    expect(percentage(-0.000000001)).toBe('-1.000e-7');
    const sparse = binomial(1, 1000000000);
    expect(sparse.report).toContain('| 1.000e-7 | % | 1 / 1000000000 |');
    expect(sparse.groups[0].rate).toBe(1e-9);
  });

  it('does not round a below-one confidence level or event rate into 100 percent certainty', () => {
    for (const level of [0.9999999, 1 - Number.EPSILON]) {
      const result = binomial(5, 10, level);
      const shown = Number(result.report.match(/approximate (\S+)% confidence/u)[1]);
      expect(shown).toBeLessThan(100);
      expect(shown).toBeGreaterThan(99.9999);
      expect(result.confidence_level).toBe(level);
    }
    expect(Number(percentage(0.9999999))).toBeLessThan(100);
    expect(Number(percentage(-0.9999999))).toBeGreaterThan(-100);
  });

  it('preserves exact money strings and units in every reported reconciliation row', () => {
    const result = total({ values: ['9007199254740993.10', '0.20', '-0.05'], unit: 'USD', expected: '9007199254740993.26' });
    expect(result.report).toContain('| Computed total | 9007199254740993.25 | USD |');
    expect(result.report).toContain('| Expected total | 9007199254740993.26 | USD |');
    expect(result.report).toContain('| Computed minus expected | -0.01 | USD |');
    expect(result.report).toContain('Summed 3 supplied values; no rounding');
    expect(result.report).toContain('not source completeness or authenticity');
    expect(total({ values: ['-0.1', '0.1'], unit: 'count' }).report).toContain('| 0.0 | count |');
  });

  it('escapes table-breaking source labels without altering machine-readable labels', () => {
    const label = 'A|B\n[click](x)<tag>_bold_';
    const result = interval({ design: 'independent-binomial', groups: [{ label, events: 1, total: 10 }] });
    expect(result.groups[0].label).toBe(label);
    expect(result.report).toContain('A\\|B \\[click\\](x)\\<tag\\>\\_bold\\_');
    expect(table(['Metric'], [['x\r\ny']])).toContain('| x y |');
    expect(table(['Metric'], [['x\ry']])).toContain('| x y |');
  });

  it('bounds large-family display explicitly while retaining every hypothesis and original ordering', () => {
    const pvalues = Array.from({ length: 21 }, (_, i) => (i + 1) / 100);
    const result = correction({ pvalues, method: 'holm' });
    expect(result.report).toContain('family-wise error rate; alpha=0.05');
    expect(result.report).toContain('Showing 20 of 21 rows');
    expect(result.report).toContain('the JSON arrays contain the complete results');
    expect(result.report).toContain('| 19 | 0.2 |');
    expect(result.report).not.toContain('| 20 | 0.21 |');
    expect(result.adjusted_pvalues).toHaveLength(21);
    expect(result.pvalues).toEqual(pvalues);
    expect(result.reject).toHaveLength(21);
  });

  it('uses accurate normal critical values rather than treating confidence level as an output score', () => {
    for (const [level, z] of [[0.90, 1.6448536269514722], [0.95, 1.959963984540054], [0.99, 2.5758293035489004]]) {
      expect(normalCritical(level)).toBeCloseTo(z, 12);
    }
  });

  it('retains sampling uncertainty with zero or all conversions', () => {
    const zero = binomial(0, 10).groups[0];
    expect(zero.rate).toBe(0);
    expect(zero.interval.low).toBe(0);
    expect(zero.interval.high).toBeCloseTo(0.2775327998628892, 12);
    const all = binomial(10, 10).groups[0];
    expect(all.interval.low).toBeCloseTo(0.7224672001371107, 12);
    expect(all.interval.high).toBe(1);
  });

  it('matches the independently published Newcombe table and preserves difference direction', () => {
    // statsmodels/stats/tests/test_proportion.py::test_confint_2indep:
    // Fagerland et al. (2015), 7/34 minus 1/34, CI [0.019, 0.340], atol .005.
    const groups = [{ events: 7, total: 34 }, { events: 1, total: 34 }];
    const result = interval({ design: 'independent-binomial', groups });
    expect(result.difference.estimate).toBeCloseTo(6 / 34, 14);
    expect(Math.abs(result.difference.interval.low - 0.019)).toBeLessThan(0.005);
    expect(Math.abs(result.difference.interval.high - 0.340)).toBeLessThan(0.005);
    const reverse = interval({ design: 'independent-binomial', groups: [...groups].reverse() });
    expect(reverse.difference.estimate).toBeCloseTo(-result.difference.estimate, 14);
    expect(reverse.difference.interval.low).toBeCloseTo(-result.difference.interval.high, 14);
    expect(reverse.difference.interval.high).toBeCloseTo(-result.difference.interval.low, 14);
  });

  it('widens intervals as requested coverage increases across sparse and large samples', () => {
    for (const n of [1, 7, 100, 1000000]) for (const events of [0, 1, Math.floor(n / 2), n]) {
      const a = binomial(events, n, 0.90).groups[0].interval;
      const b = binomial(events, n, 0.99).groups[0].interval;
      expect(b.low).toBeLessThanOrEqual(a.low + 1e-14);
      expect(b.high).toBeGreaterThanOrEqual(a.high - 1e-14);
      expect(b.low).toBeGreaterThanOrEqual(0);
      expect(b.high).toBeLessThanOrEqual(1);
      expect(b.high).toBeGreaterThan(b.low);
    }
  });

  it.each([
    { design: 'paired', groups: [{ events: 2, total: 10 }] },
    { design: 'independent-binomial', groups: [{ events: 2, total: 0 }] },
    { design: 'independent-binomial', groups: [{ events: 11, total: 10 }] },
    { design: 'independent-binomial', groups: [{ events: '2', total: 10 }] },
    { design: 'independent-binomial', groups: [{ events: 1.5, total: 10 }] },
    { design: 'independent-binomial', groups: [{ events: 2, total: 10 }], confidence_level: null },
    { design: 'independent-binomial', groups: [{ events: 2, total: 10 }], confidence_level: 1 },
    { design: 'independent-binomial', groups: [{ events: 2, total: 10, weights: [1] }] },
  ])('returns no estimate for unsupported or invalid analytical inputs: %j', async value => {
    const result = await proportions({ args: ['--json', JSON.stringify(value)] });
    expect(result.ok).toBe(false);
    expect(result.error.code).toBe('INVALID_INPUT');
    expect(result.error.message.length).toBeGreaterThan(10);
    expect(result).not.toHaveProperty('groups');
  });

  it('distinguishes family-wise and discovery-rate decisions and retains hypothesis order', () => {
    const pvalues = [0.04, 0.001, 0.03, 0.02];
    const holm = correction({ pvalues, method: 'holm' });
    expect(holm.adjusted_pvalues).toEqual([0.06, 0.004, 0.06, 0.06]);
    expect(holm.reject).toEqual([false, true, false, false]);
    const bh = correction({ pvalues, method: 'bh' });
    expect(bh.adjusted_pvalues).toEqual([0.04, 0.004, 0.04, 0.04]);
    expect(bh.reject).toEqual([true, true, true, true]);
    const by = correction({ pvalues, method: 'by' });
    expect(by.adjusted_pvalues[0]).toBeCloseTo(1 / 12, 14);
    expect(by.adjusted_pvalues[1]).toBeCloseTo(1 / 120, 14);
    expect(by.reject).toEqual([false, true, false, false]);
    expect(pvalues).toEqual([0.04, 0.001, 0.03, 0.02]);
  });

  it('correction handles ties, boundaries and permutations without reducing raw p-values', () => {
    const pvalues = [1, 0.05, 0, 0.05, 0.4];
    for (const method of ['holm', 'bh', 'by']) {
      const result = correction({ pvalues, method });
      const reverse = correction({ pvalues: [...pvalues].reverse(), method });
      expect(reverse.adjusted_pvalues.reverse()).toEqual(result.adjusted_pvalues);
      expect(result.adjusted_pvalues[1]).toBe(result.adjusted_pvalues[3]);
      result.adjusted_pvalues.forEach((p: number, i: number) => {
        expect(p).toBeGreaterThanOrEqual(pvalues[i]);
        expect(p).toBeLessThanOrEqual(1);
      });
    }
  });

  it.each([{ pvalues: [], method: 'bh' }, { pvalues: [null], method: 'holm' }, { pvalues: [1.1], method: 'bh' }, { pvalues: [0.1], method: 'auto' }, { pvalues: [0.1], method: 'holm', alpha: null }])('rejects invalid test families without inventing corrected results: %j', async value => {
    const result = await adjust({ args: ['--json', JSON.stringify(value)] });
    expect(result.ok).toBe(false);
    expect(result).not.toHaveProperty('adjusted_pvalues');
  });

  it('reconciles exact monetary totals beyond the safe-integer boundary, including refunds', () => {
    const value = { values: ['9007199254740993.10', '0.20', '-0.05'], unit: 'source cents', expected: '9007199254740993.26' };
    const result = total(value);
    expect(result).toMatchObject({ count: 3, total: '9007199254740993.25', unit: 'source cents', difference: '-0.01', matches: false });
    expect(total({ ...value, expected: result.total }).matches).toBe(true);
    const partition = total({ values: value.values.slice(0, 2), unit: value.unit });
    expect(total({ values: [partition.total, value.values[2]], unit: value.unit }).total).toBe(result.total);
    expect(value.values).toEqual(['9007199254740993.10', '0.20', '-0.05']);
  });

  it('handles empty input and signed zero without claiming input completeness', () => {
    expect(total({ values: [], unit: 'count' })).toMatchObject({ count: 0, total: '0' });
    expect(total({ values: ['-0.02', '0.020'], unit: 'unknown currency cents' }).total).toBe('0.000');
  });

  it.each([['1e3'], ['1,000'], [0.1], [null], ['2元']])('rejects lossy or ambiguous money input: %j', async values => {
    const result = await exact({ args: ['--json', JSON.stringify({ values, unit: 'cents' })] });
    expect(result.ok).toBe(false);
    expect(result.error.message).toContain('decimal strings');
    expect(result).not.toHaveProperty('total');
  });

  it('reads a BOM/Unicode-spaced input through the real runner and reports recoverable failures without writes', async () => {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'analysis calculations '));
    tempRoots.push(root);
    const source = path.join(root, '输入 amounts.json');
    const body = '\uFEFF' + JSON.stringify({ values: ['0.1', '0.2'], unit: 'cents', expected: '0.3' });
    fs.writeFileSync(source, body);
    const runner = (args: string[]) => spawnSync(process.execPath, [path.resolve('bin/run-skill.cjs'), 'data-analysis', 'exact-total', '--', ...args], {
      cwd: root, encoding: 'utf8', timeout: 30000,
      env: { ...process.env, ELECTRON_RUN_AS_NODE: '1', ORKAS_PC_DIR: process.cwd(), ORKAS_WORKSPACE_ROOT: path.join(root, 'data'), ORKAS_RUN_SKILL_DIR: path.resolve('resources/builtin/marketplace/skills/08837be6032e') },
    });
    const success = runner(['--input', source]);
    expect(success.status, success.stdout + success.stderr).toBe(0);
    expect(success.stderr).toBe('');
    expect(JSON.parse(success.stdout)).toMatchObject({ ok: true, total: '0.3', matches: true });
    const bad = runner(['--json', '{"values":[0.1],"unit":"cents"}']);
    expect(bad.status).not.toBe(0);
    expect(bad.stderr).toBe('');
    expect(JSON.parse(bad.stdout)).toMatchObject({ ok: false, error: { code: 'INVALID_INPUT' } });
    const missing = await exact({ args: ['--input', path.join(root, 'absent.json')] });
    expect(missing.error.message).toContain('Check the supplied path');
    expect(missing.error.message).not.toContain(root);
    expect(fs.readFileSync(source, 'utf8')).toBe(body);
    expect(fs.readdirSync(root)).toEqual(['输入 amounts.json']);
  });

  it('rejects oversized, malformed and unknown-option input rather than computing a partial or differently configured result', async () => {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'analysis-input-limit-'));
    tempRoots.push(root);
    const source = path.join(root, 'oversized.json');
    fs.writeFileSync(source, ' '.repeat(8 * 1024 * 1024 + 1));
    for (const args of [
      ['--input', source],
      ['--json', '{'],
      ['--json', '{}', '--confidence', '0.99'],
      ['--json', '{"values":["1"],"unit":"cents","rounding":0}'],
    ]) {
      const result = await exact({ args });
      expect(result.ok).toBe(false);
      expect(result.error.message.length).toBeGreaterThan(10);
      expect(result).not.toHaveProperty('total');
    }
    expect(fs.statSync(source).size).toBe(8 * 1024 * 1024 + 1);
  });
});