Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover zero-channel audio conversion

Louis Beaumont03d3170b4442
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index cbb41d953..5a72933c9 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,16 @@
# Coding-agent regression evals
+`app-zero-channel-downmix` compiles the actual complete converter module with
+synthetic buffers. Ten outcomes cover zero-channel passthrough and preserved
+mono, signed multichannel, silence, empty, high-channel and partial-frame behavior.
+The parent fails the two zero-channel outcomes and preserves eight; the historical
+converter fix and current source pass ten. Run
+`bun test evals/coding-agent/calibrate-zero-channel-downmix.test.js` for calibration.
+The dependency-free Cargo fixture is materialized only at grading, with no build
+cache links. This does not run the full audio crate, capture callbacks, device
+configuration gates, Windows COM or real recordings, and establishes neither
+execution isolation nor model improvement.
+
The unmarked HTML file-preview case executes ViewerFileContent with the actual
Markdown, code-renderer, iframe and sandbox helper sources. It checks full and
snippet previews, parsed content/CSP metadata, source toggling, path transitions,
@@ -67,7 +78,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 141 git-mined regressions. See
+The current app corpus contains 142 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
diff --git a/evals/coding-agent/calibrate-zero-channel-downmix.test.js b/evals/coding-agent/calibrate-zero-channel-downmix.test.js
new file mode 100644
index 000000000..d86ce7615
--- /dev/null
+++ b/evals/coding-agent/calibrate-zero-channel-downmix.test.js
@@ -0,0 +1,58 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-zero-channel-downmix');
+const path = item.oracle_paths[0];
+const git = ref => execFileSync('git', ['show', `${ref}:${path}`], { cwd: repo, encoding: 'utf8' });
+const parent = git(item.base_ref), fixed = git(item.oracle_ref);
+const hash = bytes => createHash('sha256').update(bytes).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+test('calibrate zero-channel converter outcomes and preserved samples', () => {
+ const controls = [
+ ['parent', parent, 'fail'], ['reference', fixed, 'pass'],
+ ['equivalent', change('channels.max(1)', '(if channels == 0 { 1 } else { channels })'), 'pass'],
+ ['unused-correct', parent, 'fail'],
+ ['blanket-empty', 'pub fn audio_to_mono(_: &[f32], _: u16) -> Vec<f32> { vec![] }', 'fail'],
+ ['zero-discards', change('let channels = channels.max(1) as usize;', 'if channels == 0 { return vec![]; }\n let channels = channels as usize;'), 'fail'],
+ ['zero-empty-panics', change('let channels = channels.max(1) as usize;', 'if channels == 0 && audio.is_empty() { panic!("empty buffer"); }\n let channels = channels.max(1) as usize;'), 'fail'],
+ ['no-average', change('let mono_sample = sum / channels as f32;', 'let mono_sample = sum;'), 'fail'],
+ ['drop-partial', change('audio.chunks(channels)', 'audio.chunks_exact(channels)'), 'fail'],
+ ['partial-denominator', change('let mono_sample = sum / channels as f32;', 'let mono_sample = sum / chunk.len() as f32;'), 'fail'],
+ ['rectify-samples', change('chunk.iter().sum()', 'chunk.iter().map(|sample| sample.abs()).sum()'), 'fail'],
+ ['missing-source', null, 'error'],
+ ];
+ for (const [name, source, expected] of controls) {
+ const root = mkdtempSync(join(tmpdir(), 'downmix-calibration-'));
+ try {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ if (source !== null) writeFileSync(join(root, path), source);
+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.rs'), fixed);
+ const fixtureHashes = {};
+ for (const fixture of item.grader.fixtures) {
+ const bytes = readFileSync(join(import.meta.dir, fixture.local_path));
+ const destination = join(root, fixture.destination_path);
+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes);
+ fixtureHashes[fixture.local_path] = hash(bytes);
+ }
+ const result = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd: root, encoding: 'utf8', timeout: item.grader.timeout_seconds * 1000, maxBuffer: 8 * 1024 * 1024 });
+ const errorKind = classifyGraderError(result);
+ const observed = result.error || result.signal || errorKind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, status: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: errorKind, source_sha256: source === null ? null : hash(source), fixture_hashes: fixtureHashes, stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stdout).toContain('10 passed; 0 failed');
+ if (expected === 'fail') { expect(result.status).toBe(101); expect(result.stdout).toContain('test result: FAILED.'); }
+ if (name === 'parent' || name === 'unused-correct') expect(result.stdout).toContain('8 passed; 2 failed');
+ if (name === 'missing-source') expect(errorKind).toBe('rust_compile_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+ }
+}, 240_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 48dbc4b0e..c8b93576f 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5889,6 +5889,49 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-codex-images.config.mjs"
}
+ },
+ {
+ "id": "app-zero-channel-downmix",
+ "name": "Keep degenerate audio channel metadata from crashing sample conversion",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "7c3b846ba35d6b90448b4d72b302c3fbb8f85bcc"
+ },
+ "base_ref": "e3cbb57f61c0805accedf97cf0c062880566ad68",
+ "oracle_ref": "7c3b846ba35d6b90448b4d72b302c3fbb8f85bcc",
+ "oracle_paths": [
+ "crates/screenpipe-audio/src/utils/audio/convert.rs"
+ ],
+ "trigger_paths": [
+ "crates/screenpipe-audio/src/utils/audio/convert.rs"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "audio",
+ "preserved-behavior"
+ ],
+ "prompt": "Audio conversion can panic when a device reports zero channels. Repair the existing interleaved-f32-to-mono converter so zero-channel metadata returns the original samples unchanged, including an empty buffer, without panicking. Preserve mono passthrough. For positive multichannel counts, average each frame across the declared channel count, preserve sample order and signed values, and retain real silence from cancellation. Preserve high-channel inputs and historical partial-frame behavior: emit the trailing partial frame using the full declared channel count as the denominator, even when the whole input is shorter than a frame. Empty buffers remain empty. Use the existing converter interface; no particular algorithm is required. Grading compiles the actual complete converter module with synthetic buffers. Device configuration rejection, capture callbacks, Windows COM, operating-system recovery, real recording and execution isolation are outside this narrow conversion task. Do not access recording hardware, user recordings or live services.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/zero-channel-downmix.fixture.rs",
+ "destination_path": ".eval-downmix/outcomes.rs"
+ },
+ {
+ "local_path": "graders/zero-channel-downmix.lib.rs",
+ "destination_path": ".eval-downmix/lib.rs"
+ },
+ {
+ "local_path": "graders/zero-channel-downmix.Cargo.toml.txt",
+ "destination_path": ".eval-downmix/Cargo.toml"
+ }
+ ],
+ "command": "cargo test --offline --manifest-path .eval-downmix/Cargo.toml --test outcomes -- --nocapture"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/zero-channel-downmix.Cargo.toml.txt b/evals/coding-agent/graders/zero-channel-downmix.Cargo.toml.txt
new file mode 100644
index 000000000..8313b6a3c
--- /dev/null
+++ b/evals/coding-agent/graders/zero-channel-downmix.Cargo.toml.txt
@@ -0,0 +1,12 @@
+# screenpipe — AI that knows everything you've seen, said, or heard
+# https://screenpipe.com
+[package]
+name = "downmix_probe"
+version = "0.0.0"
+edition = "2021"
+[workspace]
+[lib]
+path = "lib.rs"
+[[test]]
+name = "outcomes"
+path = "outcomes.rs"
diff --git a/evals/coding-agent/graders/zero-channel-downmix.fixture.rs b/evals/coding-agent/graders/zero-channel-downmix.fixture.rs
new file mode 100644
index 000000000..73210acec
--- /dev/null
+++ b/evals/coding-agent/graders/zero-channel-downmix.fixture.rs
@@ -0,0 +1,51 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+use downmix_probe::audio_to_mono;
+#[test]
+fn zero_nonempty() {
+ let x = [0.25, -0.5, 1.0];
+ assert_eq!(audio_to_mono(&x, 0), x);
+}
+#[test]
+fn zero_empty() {
+ assert_eq!(audio_to_mono(&[], 0), Vec::<f32>::new());
+}
+#[test]
+fn mono() {
+ let x = [-1.0, 0.0, 0.25];
+ assert_eq!(audio_to_mono(&x, 1), x);
+}
+#[test]
+fn stereo() {
+ assert_eq!(
+ audio_to_mono(&[0.5, 1.0, -0.5, 0.25], 2),
+ vec![0.75, -0.125]
+ );
+}
+#[test]
+fn four_channels() {
+ assert_eq!(
+ audio_to_mono(&[1.0, 0.0, -0.5, 0.5, 0.25, 0.25, 0.25, 0.25], 4),
+ vec![0.25, 0.25]
+ );
+}
+#[test]
+fn real_silence() {
+ assert_eq!(audio_to_mono(&[1.0, -1.0, 0.5, -0.5], 4), vec![0.0]);
+}
+#[test]
+fn partial_frame() {
+ assert_eq!(audio_to_mono(&[1.0, 1.0, 2.0], 2), vec![1.0, 1.0]);
+}
+#[test]
+fn high_channels() {
+ assert_eq!(audio_to_mono(&vec![0.5; 128], 64), vec![0.5, 0.5]);
+}
+#[test]
+fn short_frame() {
+ assert_eq!(audio_to_mono(&[1.0, 1.0], 8), vec![0.25]);
+}
+#[test]
+fn empty_valid() {
+ assert!(audio_to_mono(&[], 2).is_empty());
+}
diff --git a/evals/coding-agent/graders/zero-channel-downmix.lib.rs b/evals/coding-agent/graders/zero-channel-downmix.lib.rs
new file mode 100644
index 000000000..eb75c3c44
--- /dev/null
+++ b/evals/coding-agent/graders/zero-channel-downmix.lib.rs
@@ -0,0 +1,6 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+include!(concat!(
+ env!("CARGO_MANIFEST_DIR"),
+ "/../crates/screenpipe-audio/src/utils/audio/convert.rs"
+));