Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(pipes): run tasks when a spoken phrase is detected (#7058)

Louis Beaumontc62e2b7adf0f
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
index 637fb8a3d..9cfb86b3d 100644
--- a/apps/screenpipe-app-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
@@ -68,6 +68,35 @@ describe("PipeTriggerPicker app trigger catalog", () => {
fetchMock.mockResolvedValue(response({}));
});
+ it("saves multiple spoken phrases without requiring a connection", async () => {
+ const { applyOptimistic } = renderPicker();
+ chooseOption(/^spoken phrase/i);
+ const add = screen.getAllByRole("button", { name: "add trigger" }).at(-1)!;
+ expect(add).toBeDisabled();
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: "!!!" } });
+ expect(add).toBeDisabled();
+ for (const value of ["x".repeat(257), Array(33).fill("start job").join("\n")]) {
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value } });
+ expect(add).toBeDisabled();
+ }
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: " start job \nstop job\n" } });
+ expect(screen.getByLabelText("listen to")).toHaveValue("input");
+ clickDetailAdd();
+ const expected = { sources: [{ app: "audio", kind: "phrase", filter: { phrases: "start job\nstop job", device: "input" } }] };
+ expect(applyOptimistic).toHaveBeenCalledWith(expected);
+ await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/pipes/follow-up/config", expect.objectContaining({ body: JSON.stringify({ trigger: expected }) })));
+ expect(fetchMock.mock.calls.every(([url]) => !String(url).includes("/connections/"))).toBe(true);
+ });
+
+ it("only includes system audio when explicitly selected", () => {
+ const { applyOptimistic } = renderPicker();
+ chooseOption(/^spoken phrase/i);
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: "stop job" } });
+ fireEvent.change(screen.getByLabelText("listen to"), { target: { value: "all" } });
+ clickDetailAdd();
+ expect(applyOptimistic).toHaveBeenCalledWith({ sources: [{ app: "audio", kind: "phrase", filter: { phrases: "stop job", device: "all" } }] });
+ });
+
it("surfaces the expanded popular-app catalog", () => {
renderPicker();
diff --git a/apps/screenpipe-app-tauri/components/settings/pipe-trigger-picker.tsx b/apps/screenpipe-app-tauri/components/settings/pipe-trigger-picker.tsx
index 8f0a0a997..38c938630 100644
--- a/apps/screenpipe-app-tauri/components/settings/pipe-trigger-picker.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/pipe-trigger-picker.tsx
@@ -12,7 +12,7 @@ import { IntegrationIcon } from "@/components/settings/connections-section";
import { PipeScheduleBuilder } from "./pipe-schedule-builder";
import type { ScheduleConfig } from "@/lib/utils/schedule-builder";
import type { AvailableConnection } from "@/lib/pipe-connections";
-import { Plus, Search, Clock, CalendarClock, Workflow, Loader2, Check } from "lucide-react";
+import { Plus, Search, Clock, CalendarClock, Workflow, Loader2, Check, Mic } from "lucide-react";
export interface TriggerSource {
app: string;
@@ -64,6 +64,7 @@ type SourceApp =
| "todoist";
type OptionId =
+ | "voice_phrase"
| "schedule"
| "meeting_started"
| "meeting_ended"
@@ -97,6 +98,7 @@ const OPTIONS: Option[] = [
// "cron" is implementation vocabulary, not the user's — the picker offers
// plain cadences and keeps the raw expression as an advanced escape hatch.
{ id: "schedule", group: "recurring", label: "on a schedule", sub: "hourly, daily, every N minutes" },
+ { id: "voice_phrase", group: "voice", label: "spoken phrase", sub: "when you say a word or phrase" },
{ id: "meeting_started", group: "meetings", label: "meeting starts", sub: "a call is detected" },
{ id: "meeting_ended", group: "meetings", label: "meeting ends", sub: "a call wraps up" },
{ id: "email_received", group: "email", label: "new email", sub: "Gmail or any IMAP inbox", app: "imap", icon: "gmail", kind: "message" },
@@ -115,10 +117,11 @@ const OPTIONS: Option[] = [
{ id: "obsidian", group: "obsidian", label: "new note", sub: "in a vault folder", app: "obsidian" },
{ id: "pipe", group: "pipes", label: "after a scheduled task finishes", sub: "chain off another scheduled task" },
];
-const GROUP_ORDER = ["recurring", "meetings", "email", "calendar", "slack", "notion", "github", "linear", "todoist", "obsidian", "pipes"];
+const GROUP_ORDER = ["recurring", "voice", "meetings", "email", "calendar", "slack", "notion", "github", "linear", "todoist", "obsidian", "pipes"];
function optionIcon(o: Option) {
if (o.app) return <IntegrationIcon icon={o.icon || o.app} className="w-4 h-4 flex items-center justify-center" fallbackClassName="h-4 w-4 text-muted-foreground" />;
+ if (o.id === "voice_phrase") return <Mic className="h-4 w-4 text-muted-foreground" />;
if (o.id === "schedule") return <Clock className="h-4 w-4 text-muted-foreground" />;
if (o.id === "pipe") return <Workflow className="h-4 w-4 text-muted-foreground" />;
return <CalendarClock className="h-4 w-4 text-muted-foreground" />;
@@ -133,6 +136,7 @@ function eventLabel(e: string): string {
return e.replace(/_/g, " ");
}
function sourceLabel(s: TriggerSource): string {
+ if (s.app === "audio") return `when ${s.filter?.device === "all" ? "audio contains" : "you say"} ${(s.filter?.phrases || "").split("\n").filter(Boolean).map((p) => `“${p}”`).join(" or ")}`;
const acct = s.instance ? ` (${s.instance})` : "";
if (s.app === "slack") return `slack${acct} · ${s.filter?.channel_name || s.filter?.channel || "a channel"}`;
if (s.app === "notion") return `notion${acct} · ${s.filter?.database_name || "any page edited"}`;
@@ -383,6 +387,7 @@ function Detail({
onAdd={() => onAddEvent(option.id)}
/>
)}
+ {option.id === "voice_phrase" && <VoicePhraseDetail onAdd={onAddSource} />}
{option.id === "pipe" && <PipeDetail pipes={otherPipes} onAdd={(name) => onAddEvent(`pipe_completed:${name}`)} />}
{option.app && (
<SourceDetail
@@ -401,6 +406,7 @@ function Detail({
function detailTitle(id: OptionId): string {
switch (id) {
+ case "voice_phrase": return "when a phrase is spoken";
case "schedule": return "on a schedule";
case "meeting_started": return "when a meeting starts";
case "meeting_ended": return "when a meeting ends";
@@ -439,6 +445,37 @@ function SimpleDetail({ text, onAdd }: { text: string; onAdd: () => void }) {
);
}
+function VoicePhraseDetail({ onAdd }: { onAdd: (s: TriggerSource) => void }) {
+ const [phrases, setPhrases] = useState("");
+ const [device, setDevice] = useState("input");
+ const cleaned = phrases.split("\n").map((p) => p.trim()).filter(Boolean);
+ const valid = cleaned.length > 0 && cleaned.length <= 32
+ && cleaned.every((p) => [...p].length <= 256 && /[\p{L}\p{N}]/u.test(p));
+ return (
+ <div>
+ <p className="text-xs text-muted-foreground mb-3">
+ Run when a new transcript contains any phrase below. Matching ignores case and punctuation.
+ The task receives the matching words and their timestamp.
+ </p>
+ <label htmlFor="voice-trigger-phrases" className={LABEL}>words or phrases, one per line</label>
+ <textarea id="voice-trigger-phrases" value={phrases} onChange={(e) => setPhrases(e.target.value)}
+ maxLength={32768} placeholder={"start job\nstop job"} className={`${INPUT.replace("h-9", "h-20")} py-2 mt-1`} />
+ <p className="text-[11px] text-muted-foreground mt-1">Up to 32 phrases, 256 characters each. Each phrase must fit in one transcript segment.</p>
+ <label htmlFor="voice-trigger-device" className={`${LABEL} block mt-3`}>listen to</label>
+ <select id="voice-trigger-device" value={device} onChange={(e) => setDevice(e.target.value)} className={`${INPUT} mt-1`}>
+ <option value="input">microphone</option>
+ <option value="all">microphone and system audio</option>
+ </select>
+ <p className="text-[11px] text-muted-foreground mt-3">
+ Audio recording and transcription must be on. Checks new transcripts every 30 seconds;
+ transcription may add a delay. Phrase matching uses no AI credits. Task runs use your selected model.
+ Existing schedules still run separately. Use manual scheduling to run only on triggers.
+ </p>
+ <PrimaryAdd disabled={!valid} onClick={() => onAdd({ app: "audio", kind: "phrase", filter: { phrases: cleaned.join("\n"), device } })} />
+ </div>
+ );
+}
+
function PipeDetail({ pipes, onAdd }: { pipes: { name: string }[]; onAdd: (name: string) => void }) {
const [name, setName] = useState("");
if (!pipes.length) return <p className="text-xs text-muted-foreground">No other enabled scheduled tasks yet — create one first.</p>;
diff --git a/apps/screenpipe-workflows-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx b/apps/screenpipe-workflows-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
index 637fb8a3d..9cfb86b3d 100644
--- a/apps/screenpipe-workflows-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
+++ b/apps/screenpipe-workflows-tauri/components/settings/__tests__/pipe-trigger-picker.test.tsx
@@ -68,6 +68,35 @@ describe("PipeTriggerPicker app trigger catalog", () => {
fetchMock.mockResolvedValue(response({}));
});
+ it("saves multiple spoken phrases without requiring a connection", async () => {
+ const { applyOptimistic } = renderPicker();
+ chooseOption(/^spoken phrase/i);
+ const add = screen.getAllByRole("button", { name: "add trigger" }).at(-1)!;
+ expect(add).toBeDisabled();
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: "!!!" } });
+ expect(add).toBeDisabled();
+ for (const value of ["x".repeat(257), Array(33).fill("start job").join("\n")]) {
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value } });
+ expect(add).toBeDisabled();
+ }
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: " start job \nstop job\n" } });
+ expect(screen.getByLabelText("listen to")).toHaveValue("input");
+ clickDetailAdd();
+ const expected = { sources: [{ app: "audio", kind: "phrase", filter: { phrases: "start job\nstop job", device: "input" } }] };
+ expect(applyOptimistic).toHaveBeenCalledWith(expected);
+ await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/pipes/follow-up/config", expect.objectContaining({ body: JSON.stringify({ trigger: expected }) })));
+ expect(fetchMock.mock.calls.every(([url]) => !String(url).includes("/connections/"))).toBe(true);
+ });
+
+ it("only includes system audio when explicitly selected", () => {
+ const { applyOptimistic } = renderPicker();
+ chooseOption(/^spoken phrase/i);
+ fireEvent.change(screen.getByLabelText("words or phrases, one per line"), { target: { value: "stop job" } });
+ fireEvent.change(screen.getByLabelText("listen to"), { target: { value: "all" } });
+ clickDetailAdd();
+ expect(applyOptimistic).toHaveBeenCalledWith({ sources: [{ app: "audio", kind: "phrase", filter: { phrases: "stop job", device: "all" } }] });
+ });
+
it("surfaces the expanded popular-app catalog", () => {
renderPicker();
diff --git a/apps/screenpipe-workflows-tauri/components/settings/pipe-trigger-picker.tsx b/apps/screenpipe-workflows-tauri/components/settings/pipe-trigger-picker.tsx
index 15bea0b5f..a5efaa11c 100644
--- a/apps/screenpipe-workflows-tauri/components/settings/pipe-trigger-picker.tsx
+++ b/apps/screenpipe-workflows-tauri/components/settings/pipe-trigger-picker.tsx
@@ -12,7 +12,7 @@ import { IntegrationIcon } from "@/components/settings/connections-section";
import { PipeScheduleBuilder } from "./pipe-schedule-builder";
import type { ScheduleConfig } from "@/lib/utils/schedule-builder";
import type { AvailableConnection } from "@/lib/pipe-connections";
-import { Plus, Search, Clock, CalendarClock, Workflow, Loader2, Check } from "lucide-react";
+import { Plus, Search, Clock, CalendarClock, Workflow, Loader2, Check, Mic } from "lucide-react";
export interface TriggerSource {
app: string;
@@ -64,6 +64,7 @@ type SourceApp =
| "todoist";
type OptionId =
+ | "voice_phrase"
| "schedule"
| "meeting_started"
| "meeting_ended"
@@ -97,6 +98,7 @@ const OPTIONS: Option[] = [
// "cron" is implementation vocabulary, not the user's — the picker offers
// plain cadences and keeps the raw expression as an advanced escape hatch.
{ id: "schedule", group: "recurring", label: "on a schedule", sub: "hourly, daily, every N minutes" },
+ { id: "voice_phrase", group: "voice", label: "spoken phrase", sub: "when you say a word or phrase" },
{ id: "meeting_started", group: "meetings", label: "meeting starts", sub: "a call is detected" },
{ id: "meeting_ended", group: "meetings", label: "meeting ends", sub: "a call wraps up" },
{ id: "email_received", group: "email", label: "new email", sub: "Gmail or any IMAP inbox", app: "imap", icon: "gmail", kind: "message" },
@@ -115,10 +117,11 @@ const OPTIONS: Option[] = [
{ id: "obsidian", group: "obsidian", label: "new note", sub: "in a vault folder", app: "obsidian" },
{ id: "pipe", group: "pipes", label: "after a scheduled task finishes", sub: "chain off another scheduled task" },
];
-const GROUP_ORDER = ["recurring", "meetings", "email", "calendar", "slack", "notion", "github", "linear", "todoist", "obsidian", "pipes"];
+const GROUP_ORDER = ["recurring", "voice", "meetings", "email", "calendar", "slack", "notion", "github", "linear", "todoist", "obsidian", "pipes"];
function optionIcon(o: Option) {
if (o.app) return <IntegrationIcon icon={o.icon || o.app} className="w-4 h-4 flex items-center justify-center" fallbackClassName="h-4 w-4 text-muted-foreground" />;
+ if (o.id === "voice_phrase") return <Mic className="h-4 w-4 text-muted-foreground" />;
if (o.id === "schedule") return <Clock className="h-4 w-4 text-muted-foreground" />;
if (o.id === "pipe") return <Workflow className="h-4 w-4 text-muted-foreground" />;
return <CalendarClock className="h-4 w-4 text-muted-foreground" />;
@@ -133,6 +136,7 @@ function eventLabel(e: string): string {
return e.replace(/_/g, " ");
}
function sourceLabel(s: TriggerSource): string {
+ if (s.app === "audio") return `when ${s.filter?.device === "all" ? "audio contains" : "you say"} ${(s.filter?.phrases || "").split("\n").filter(Boolean).map((p) => `“${p}”`).join(" or ")}`;
const acct = s.instance ? ` (${s.instance})` : "";
if (s.app === "slack") return `slack${acct} · ${s.filter?.channel_name || s.filter?.channel || "a channel"}`;
if (s.app === "notion") return `notion${acct} · ${s.filter?.database_name || "any page edited"}`;
@@ -383,6 +387,7 @@ function Detail({
onAdd={() => onAddEvent(option.id)}
/>
)}
+ {option.id === "voice_phrase" && <VoicePhraseDetail onAdd={onAddSource} />}
{option.id === "pipe" && <PipeDetail pipes={otherPipes} onAdd={(name) => onAddEvent(`pipe_completed:${name}`)} />}
{option.app && (
<SourceDetail
@@ -401,6 +406,7 @@ function Detail({
function detailTitle(id: OptionId): string {
switch (id) {
+ case "voice_phrase": return "when a phrase is spoken";
case "schedule": return "on a schedule";
case "meeting_started": return "when a meeting starts";
case "meeting_ended": return "when a meeting ends";
@@ -439,6 +445,37 @@ function SimpleDetail({ text, onAdd }: { text: string; onAdd: () => void }) {
);
}
+function VoicePhraseDetail({ onAdd }: { onAdd: (s: TriggerSource) => void }) {
+ const [phrases, setPhrases] = useState("");
+ const [device, setDevice] = useState("input");
+ const cleaned = phrases.split("\n").map((p) => p.trim()).filter(Boolean);
+ const valid = cleaned.length > 0 && cleaned.length <= 32
+ && cleaned.every((p) => [...p].length <= 256 && /[\p{L}\p{N}]/u.test(p));
+ return (
+ <div>
+ <p className="text-xs text-muted-foreground mb-3">
+ Run when a new transcript contains any phrase below. Matching ignores case and punctuation.
+ The task receives the matching words and their timestamp.
+ </p>
+ <label htmlFor="voice-trigger-phrases" className={LABEL}>words or phrases, one per line</label>
+ <textarea id="voice-trigger-phrases" value={phrases} onChange={(e) => setPhrases(e.target.value)}
+ maxLength={32768} placeholder={"start job\nstop job"} className={`${INPUT.replace("h-9", "h-20")} py-2 mt-1`} />
+ <p className="text-[11px] text-muted-foreground mt-1">Up to 32 phrases, 256 characters each. Each phrase must fit in one transcript segment.</p>
+ <label htmlFor="voice-trigger-device" className={`${LABEL} block mt-3`}>listen to</label>
+ <select id="voice-trigger-device" value={device} onChange={(e) => setDevice(e.target.value)} className={`${INPUT} mt-1`}>
+ <option value="input">microphone</option>
+ <option value="all">microphone and system audio</option>
+ </select>
+ <p className="text-[11px] text-muted-foreground mt-3">
+ Audio recording and transcription must be on. Checks new transcripts every 30 seconds;
+ transcription may add a delay. Phrase matching uses no AI credits. Task runs use your selected model.
+ Existing schedules still run separately. Use manual scheduling to run only on triggers.
+ </p>
+ <PrimaryAdd disabled={!valid} onClick={() => onAdd({ app: "audio", kind: "phrase", filter: { phrases: cleaned.join("\n"), device } })} />
+ </div>
+ );
+}
+
function PipeDetail({ pipes, onAdd }: { pipes: { name: string }[]; onAdd: (name: string) => void }) {
const [name, setName] = useState("");
if (!pipes.length) return <p className="text-xs text-muted-foreground">No other enabled scheduled tasks yet — create one first.</p>;
diff --git a/crates/screenpipe-core/benches/voice_trigger_sql.py b/crates/screenpipe-core/benches/voice_trigger_sql.py
new file mode 100644
index 000000000..2a4e6d9a3
--- /dev/null
+++ b/crates/screenpipe-core/benches/voice_trigger_sql.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# screenpipe — AI that knows everything you've seen, said, or heard
+# https://screenpipe.com
+"""Disposable WAL probe using the production SELECTs and 1M synthetic rows.
+Run: python3 crates/screenpipe-core/benches/voice_trigger_sql.py
+Minimal schema, standard-library SQLite. Does not measure HTTP, SQLx, capture,
+FTS indexing or hybrid-storage coordination. Never accesses a live user DB.
+"""
+import json
+from pathlib import Path
+import re
+import sqlite3
+import tempfile
+import threading
+import time
+
+source = (Path(__file__).resolve().parents[1] / "src/pipes/connection_triggers/voice.rs").read_text()
+cap = int(re.search(r"MAX_TRANSCRIPT_CHARS: usize = ([\d_]+)", source)[1].replace("_", ""))
+query = re.search(r'format!\("(SELECT id, timestamp, substr[^"\n]+)"\)', source)[1]
+query = query.replace("{MAX_TRANSCRIPT_CHARS}", str(cap)).replace("{read_chars}", str(cap + 1)).replace("{after}", "?")
+baseline = re.search(r'"(SELECT COALESCE\(MAX\(id\), 0\)[^"\n]+)"', source)[1]
+rows, iterations = 1_000_000, 2000
+
+
+def summary(values):
+ ordered = sorted(values)
+ return {"samples": len(values), "p50_ms": round(ordered[len(ordered)//2], 3),
+ "p95_ms": round(ordered[int(len(ordered)*.95)], 3), "max_ms": round(max(values), 3)}
+
+
+with tempfile.TemporaryDirectory(prefix="voice-sql-probe-") as temp:
+ path = str(Path(temp) / "probe.db")
+ setup = sqlite3.connect(path)
+ setup.execute("PRAGMA journal_mode=WAL")
+ setup.execute("CREATE TABLE audio_transcriptions (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT, transcription TEXT, is_input_device INTEGER)")
+ setup.executemany("INSERT INTO audio_transcriptions(timestamp, transcription, is_input_device) VALUES('2026-09-17', ?, ?)",
+ (("ordinary recorded words " * 5, i % 2) for i in range(rows)))
+ setup.execute("UPDATE audio_transcriptions SET transcription=? WHERE id > ?", ("x" * (cap * 2), rows - 50))
+ setup.commit()
+ setup.close()
+ reader = sqlite3.connect(path, timeout=.1)
+ reader.execute("PRAGMA query_only=ON")
+ plan = [row[3] for row in reader.execute("EXPLAIN QUERY PLAN " + query, (rows - 50,))]
+ assert any("INTEGER PRIMARY KEY" in row and "rowid>?" in row for row in plan), plan
+ assert not any("TEMP B-TREE" in row for row in plan), plan
+ baseline_plan = [row[3] for row in reader.execute("EXPLAIN QUERY PLAN " + baseline)]
+ baseline_ms = []
+ for _ in range(100):
+ start = time.perf_counter()
+ assert reader.execute(baseline).fetchone()[0] == rows
+ baseline_ms.append((time.perf_counter() - start) * 1000)
+ writer_alone_ms = []
+ alone = sqlite3.connect(path, timeout=.1, isolation_level=None)
+ for _ in range(iterations):
+ start = time.perf_counter()
+ alone.execute("INSERT INTO audio_transcriptions(timestamp, transcription, is_input_device) VALUES('2026-09-17', 'start job Acme', 1)")
+ writer_alone_ms.append((time.perf_counter() - start) * 1000)
+ alone.close()
+ errors, writer_ms, scan_ms = [], [], []
+ ready = threading.Event()
+
+ def write():
+ conn = sqlite3.connect(path, timeout=.1, isolation_level=None)
+ ready.wait()
+ try:
+ for _ in range(iterations):
+ start = time.perf_counter()
+ conn.execute("INSERT INTO audio_transcriptions(timestamp, transcription, is_input_device) VALUES('2026-09-17', 'start job Acme', 1)")
+ writer_ms.append((time.perf_counter() - start) * 1000)
+ except sqlite3.Error as error:
+ errors.append(str(error))
+ finally:
+ conn.close()
+
+ worker = threading.Thread(target=write)
+ worker.start()
+ ready.set()
+ try:
+ for _ in range(iterations):
+ start = time.perf_counter()
+ batch = reader.execute(query, (rows - 50,)).fetchall()
+ scan_ms.append((time.perf_counter() - start) * 1000)
+ assert len(batch) == 50 and all(len(row[2]) == cap + 1 for row in batch)
+ except sqlite3.Error as error:
+ errors.append(str(error))
+ worker.join(timeout=30)
+ assert not worker.is_alive(), "writer exceeded 30 seconds"
+ assert not errors, errors
+ assert len(writer_ms) == iterations
+ reader.close()
+ print(json.dumps({"sqlite_version": sqlite3.sqlite_version, "rows": rows,
+ "oversized_rows": 50, "returned_char_cap": cap + 1, "query_plan": plan,
+ "baseline_plan": baseline_plan, "baseline": summary(baseline_ms),
+ "writes_alone": summary(writer_alone_ms), "scan_during_writes": summary(scan_ms), "writes_during_scans": summary(writer_ms),
+ "sqlite_errors": errors}, indent=2))
diff --git a/crates/screenpipe-core/src/pipes/connection_triggers.rs b/crates/screenpipe-core/src/pipes/connection_triggers.rs
index a188889e9..81a42c682 100644
--- a/crates/screenpipe-core/src/pipes/connection_triggers.rs
+++ b/crates/screenpipe-core/src/pipes/connection_triggers.rs
@@ -9,7 +9,8 @@
//! `connection_trigger` event addressed to the matched pipe. The watcher only
//! *detects + emits*; the scheduler only *matches + runs*. They meet at the bus.
//!
-//! Three ingestion classes, one cursor model:
+//! Ingestion classes sharing one cursor and event-delivery model:
+//! - **local audio**: one bounded read shared across all phrase subscriptions.
//! - **file** (Obsidian): scan a vault folder for new/changed `.md` files.
//! - **api poll** (Slack, Notion, email, Calendar, GitHub, Linear, Todoist): page the local connection proxy
//! (`/connections/<id>/...`, which injects auth server-side) and diff the
@@ -70,9 +71,11 @@ const RETRY_CAP: u32 = 5;
const INFLIGHT_TIMEOUT: Duration = Duration::from_secs(600);
const CURSOR_FILE: &str = ".connection-triggers.json";
-const TRIGGER_CONTEXT_FILE: &str = ".trigger-context.json";
+
+mod voice;
const SUPPORTED_APPS: &[&str] = &[
+ "audio",
"obsidian",
"slack",
"notion",
@@ -105,6 +108,9 @@ pub struct CursorState {
/// In-memory record of a fire awaiting its pipe run to complete.
#[derive(Debug, Clone)]
pub struct Pending {
+ pub delivery_id: String,
+ /// Immutable payload: retries must not absorb newly recorded items.
+ pub context: Value,
pub pipe: String,
/// Watermark this fire would commit on success.
pub token: String,
@@ -162,6 +168,9 @@ pub struct DetectedItem {
pub preview: String,
/// Source timestamp token for this item (used to advance the cursor).
pub ts: String,
+ /// Audio source metadata; omitted for connected-app items.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub is_input: Option<bool>,
}
/// Everything the API-poll sources need to reach the local connection proxy.
@@ -211,6 +220,12 @@ impl SourceCtx<'_> {
/// instant rather than compare strings — a smoke test against live Notion showed
/// offset-form timestamps. An unparseable token sorts lowest.
fn token_cmp(app: &str, a: &str, b: &str) -> Ordering {
+ if app == "audio" {
+ return a
+ .parse::<i64>()
+ .unwrap_or(-1)
+ .cmp(&b.parse::<i64>().unwrap_or(-1));
+ }
if app == "notion" {
match (
chrono::DateTime::parse_from_rfc3339(a),
@@ -305,6 +320,7 @@ fn effective_kind(src: &SourceTrigger) -> &str {
fn default_kind(app: &str) -> &str {
match app {
+ "audio" => "phrase",
"obsidian" => "note",
"slack" => "message",
"notion" => "page",
@@ -404,6 +420,7 @@ async fn fetch_items(
since: &str,
) -> Option<Vec<DetectedItem>> {
match src.app.as_str() {
+ "audio" => voice::fetch(ctx, src, since).await,
"obsidian" => fetch_obsidian(src, since).await,
"slack" => fetch_slack(ctx, src, since).await,
"notion" => fetch_notion(ctx, src, since).await,
@@ -920,6 +937,7 @@ pub fn scan_new_files(root: &Path, since_ms: u64) -> (Vec<DetectedItem>, u64) {
}
if mtime_ms > since_ms {
out.push(DetectedItem {
+ is_input: None,
id: entry.path().to_string_lossy().to_string(),
title: name,
preview: String::new(),
@@ -946,6 +964,7 @@ pub fn parse_slack_messages(value: &Value) -> Vec<(f64, DetectedItem)> {
Some((
ts_num,
DetectedItem {
+ is_input: None,
id: ts_str.to_string(),
title: first_line(text, 80),
preview: text.to_string(),
@@ -974,6 +993,7 @@ pub fn parse_notion_results(value: &Value) -> Vec<(String, DetectedItem)> {
Some((
edited.to_string(),
DetectedItem {
+ is_input: None,
id: id.to_string(),
title: extract_notion_title(obj),
preview: url.to_string(),
@@ -1017,6 +1037,7 @@ pub fn parse_imap_messages(value: &Value) -> Vec<DetectedItem> {
.or_else(|| message.get("from").and_then(Value::as_str))
.unwrap_or("");
Some(DetectedItem {
+ is_input: None,
id: uid.clone(),
title: first_line(subject, 120),
preview: peer.to_string(),
@@ -1054,6 +1075,7 @@ pub fn parse_google_calendar_events(value: &Value) -> Vec<DetectedItem> {
.or_else(|| event.get("hangoutLink").and_then(Value::as_str))
.unwrap_or("");
Some(DetectedItem {
+ is_input: None,
id: id.to_string(),
title: first_line(title, 120),
preview: preview.to_string(),
@@ -1083,6 +1105,7 @@ pub fn parse_outlook_messages(value: &Value, timestamp_field: &str) -> Vec<Detec
.and_then(Value::as_str)
.unwrap_or("");
Some(DetectedItem {
+ is_input: None,
id: id.to_string(),
title: first_line(title, 120),
preview: preview.to_string(),
@@ -1116,6 +1139,7 @@ pub fn parse_github_issues(value: &Value, kind: &str) -> Vec<DetectedItem> {
});
let preview = issue.get("html_url").and_then(Value::as_str).unwrap_or("");
Some(DetectedItem {
+ is_input: None,
id,
title: first_line(title, 120),
preview: preview.to_string(),
@@ -1156,6 +1180,7 @@ pub fn parse_todoist_tasks(value: &Value) -> Vec<DetectedItem> {
.and_then(Value::as_str)
.unwrap_or("");
Some(DetectedItem {
+ is_input: None,
id,
title: first_line(content, 120),
preview: preview.to_string(),
@@ -1207,6 +1232,7 @@ pub fn parse_linear_issues(value: &Value, kind: &str) -> Vec<DetectedItem> {
continue;
};
out.push(DetectedItem {
+ is_input: None,
id: issue_id.to_string(),
title,
preview: url.to_string(),
@@ -1253,6 +1279,7 @@ pub fn parse_linear_issues(value: &Value, kind: &str) -> Vec<DetectedItem> {
format!("assigned to {assignee} · {url}")
};
out.push(DetectedItem {
+ is_input: None,
id: history_id.to_string(),
title: title.clone(),
preview,
@@ -1286,6 +1313,7 @@ pub fn parse_linear_issues(value: &Value, kind: &str) -> Vec<DetectedItem> {
format!("{from_state} → {to_state} · {url}")
};
out.push(DetectedItem {
+ is_input: None,
id: history_id.to_string(),
title: title.clone(),
preview,
@@ -1389,13 +1417,18 @@ fn decide(
}
}
-/// Apply a pipe-run completion to any pending fires for that pipe. Returns true
+/// Acknowledge only the source delivery that actually started this execution. Returns true
/// if a committed cursor changed (needs persisting). Pure → unit-tested.
-fn apply_completion(state: &mut WatcherState, pipe: &str, success: bool) -> bool {
+fn apply_completion(
+ state: &mut WatcherState,
+ pipe: &str,
+ success: bool,
+ delivery_id: Option<&str>,
+) -> bool {
let keys: Vec<String> = state
.pending
.iter()
- .filter(|(_, p)| p.pipe == pipe)
+ .filter(|(_, p)| p.pipe == pipe && delivery_id == Some(p.delivery_id.as_str()))
.map(|(k, _)| k.clone())
.collect();
let mut dirty = false;
@@ -1432,13 +1465,35 @@ fn commit(state: &mut WatcherState, key: &str, token: &str) {
}
/// Retry pending fires that have been in flight too long with no completion seen.
-fn expire_timeouts(state: &mut WatcherState) {
- for p in state.pending.values_mut() {
- if !p.failed && p.since.elapsed() >= INFLIGHT_TIMEOUT {
+fn expire_timeouts(state: &mut WatcherState, pipes: &[(String, PipeConfig)]) {
+ let mut abandoned = Vec::new();
+ for (key, p) in &mut state.pending {
+ let run_timeout = pipes
+ .iter()
+ .find(|(name, _)| name == &p.pipe)
+ .map(|(_, config)| config.timeout.unwrap_or(super::DEFAULT_TIMEOUT_SECS))
+ .unwrap_or(super::DEFAULT_TIMEOUT_SECS);
+ let timeout = INFLIGHT_TIMEOUT.max(Duration::from_secs(
+ run_timeout.saturating_add(2 * POLL_INTERVAL_SECS),
+ ));
+ if !p.failed && p.since.elapsed() >= timeout {
p.attempts += 1;
- p.failed = true;
+ if p.attempts >= RETRY_CAP {
+ abandoned.push((key.clone(), p.token.clone()));
+ } else {
+ p.failed = true;
+ }
}
}
+ for (key, token) in abandoned {
+ warn!(
+ "connection trigger: abandoning timed-out delivery after {} attempts",
+ RETRY_CAP
+ );
+ commit(state, &key, &token);
+ state.pending.remove(&key);
+ state.dirty = true;
+ }
}
// ---------------------------------------------------------------------------
@@ -1446,21 +1501,21 @@ fn expire_timeouts(state: &mut WatcherState) {
// ---------------------------------------------------------------------------
/// One poll across every enabled pipe's sources. `completions` are
-/// `(pipe_name, success)` drained from `pipe_completed:*` since the last tick.
+/// `(pipe_name, success, delivery_id)` drained from `pipe_completed:*` since the last tick.
pub async fn poll_once(
pipes_dir: &Path,
pipes: &[(String, PipeConfig)],
state: &mut WatcherState,
ctx: &SourceCtx<'_>,
- completions: &[(String, bool)],
+ completions: &[(String, bool, Option<String>)],
) {
// 1. confirm/retire in-flight fires, then time out stuck ones.
- for (pipe, success) in completions {
- if apply_completion(state, pipe, *success) {
+ for (pipe, success, delivery_id) in completions {
+ if apply_completion(state, pipe, *success, delivery_id.as_deref()) {
state.dirty = true;
}
}
- expire_timeouts(state);
+ expire_timeouts(state, pipes);
// 2. gather subscriptions, grouped by source identity (dedup the fetch).
let mut active: HashSet<String> = HashSet::new();
@@ -1479,8 +1534,20 @@ pub async fn poll_once(
}
let key = subscription_key(pipe, src);
active.insert(key.clone());
+ // An uncompleted delivery needs no more reads. New records remain
+ // behind its committed cursor until this batch finishes.
+ if src.app == "audio" && !voice::valid_source(src) {
+ continue;
+ }
+ if retry_or_wait(state, pipe, src, &key) {
+ continue;
+ }
groups
- .entry(source_identity(src))
+ .entry(if src.app == "audio" {
+ "audio".into()
+ } else {
+ source_identity(src)
+ })
.or_default()
.push((pipe.clone(), src.clone(), key));
}
@@ -1509,8 +1576,26 @@ pub async fn poll_once(
None => continue,
};
+ // New voice subscriptions baseline at the latest insertion, even when
+ // sharing a source with a subscriber that is still draining old rows.
+ let voice_baseline = if app == "audio"
+ && !min_since.is_empty()
+ && subs
+ .iter()
+ .any(|(_, _, k)| !state.committed.contains_key(k))
+ {
+ voice::fetch(ctx, &subs[0].1, "").await
+ } else {
+ None
+ };
for (pipe, src, key) in subs {
- process_subscriber(pipes_dir, state, pipe, src, key, &raw);
+ if app == "audio" && !min_since.is_empty() && !state.committed.contains_key(key) {
+ if let Some(baseline) = voice_baseline.as_ref() {
+ process_subscriber(state, pipe, src, key, baseline);
+ }
+ continue;
+ }
+ process_subscriber(state, pipe, src, key, &raw);
}
}
@@ -1524,14 +1609,41 @@ pub async fn poll_once(
state.save(pipes_dir);
}
+/// Pending deliveries own their snapshot and never need another source read.
+fn retry_or_wait(state: &mut WatcherState, pipe: &str, src: &SourceTrigger, key: &str) -> bool {
+ let Some(pending) = state.pending.get_mut(key) else {
+ return false;
+ };
+ if pending.failed {
+ let count = pending
+ .context
+ .get("count")
+ .and_then(Value::as_u64)
+ .unwrap_or(0) as usize;
+ emit_event(
+ pipe,
+ src,
+ count,
+ key,
+ &pending.delivery_id,
+ pending.context.clone(),
+ );
+ pending.failed = false;
+ pending.since = Instant::now();
+ }
+ true
+}
+
fn process_subscriber(
- pipes_dir: &Path,
state: &mut WatcherState,
pipe: &str,
src: &SourceTrigger,
key: &str,
raw: &[DetectedItem],
) {
+ if retry_or_wait(state, pipe, src, key) {
+ return;
+ }
let app = src.app.as_str();
let committed = state.committed.get(key).cloned();
let pending = state.pending.get(key).cloned();
@@ -1558,12 +1670,31 @@ fn process_subscriber(
token,
attempts,
} => {
+ // Advance over nonmatching transcripts without launching an agent.
+ // Keep the scanned watermark, including trailing nonmatches, so a
+ // successful run never reprocesses the same recorded words.
+ let items = if app == "audio" {
+ voice::matching_items(src, items)
+ } else {
+ items
+ };
+ if items.is_empty() {
+ commit(state, key, &token);
+ state.dirty = true;
+ return;
+ }
let count = items.len();
- write_trigger_context(&pipes_dir.join(pipe), src, &items);
- emit_event(pipe, src, count);
+ let delivery_id = pending
+ .as_ref()
+ .map(|p| p.delivery_id.clone())
+ .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
+ let context = trigger_context(src, &items);
+ emit_event(pipe, src, count, key, &delivery_id, context.clone());
state.pending.insert(
key.to_string(),
Pending {
+ delivery_id,
+ context,
pipe: pipe.to_string(),
token,
attempts,
@@ -1582,26 +1713,23 @@ fn process_subscriber(
}
}
-fn write_trigger_context(pipe_dir: &Path, src: &SourceTrigger, items: &[DetectedItem]) {
- if !pipe_dir.is_dir() {
- return;
- }
- let ctx = serde_json::json!({
- "app": src.app,
- "kind": effective_kind(src),
- "instance": src.instance,
- "path": src.path,
- "filter": src.filter,
+fn trigger_context(src: &SourceTrigger, items: &[DetectedItem]) -> Value {
+ serde_json::json!({
+ "app": src.app, "kind": effective_kind(src), "instance": src.instance,
+ "path": src.path, "filter": src.filter,
"detected_at": chrono::Utc::now().to_rfc3339(),
- "count": items.len(),
- "items": items,
- });
- if let Ok(s) = serde_json::to_string_pretty(&ctx) {
- let _ = super::atomic_write(&pipe_dir.join(TRIGGER_CONTEXT_FILE), &s);
- }
+ "count": items.len(), "items": items,
+ })
}
-fn emit_event(pipe: &str, src: &SourceTrigger, count: usize) {
+fn emit_event(
+ pipe: &str,
+ src: &SourceTrigger,
+ count: usize,
+ key: &str,
+ delivery_id: &str,
+ context: Value,
+) {
let event = screenpipe_events::ConnectionTriggerEvent {
pipe: pipe.to_string(),
app: src.app.clone(),
@@ -1609,6 +1737,9 @@ fn emit_event(pipe: &str, src: &SourceTrigger, count: usize) {
path: src.path.clone(),
count,
timestamp: chrono::Utc::now(),
+ delivery_id: Some(delivery_id.to_string()),
+ subscription_key: Some(key.to_string()),
+ context: Some(context),
};
if let Err(e) = screenpipe_events::send_event("connection_trigger", event) {
warn!(
@@ -1625,6 +1756,7 @@ mod tests {
fn item(ts: &str) -> DetectedItem {
DetectedItem {
+ is_input: None,
id: ts.into(),
title: "t".into(),
preview: String::new(),
@@ -1639,6 +1771,8 @@ mod tests {
}
fn pending(pipe: &str, token: &str, attempts: u32, failed: bool) -> Pending {
Pending {
+ delivery_id: "test-delivery".into(),
+ context: serde_json::json!({"count": 1}),
pipe: pipe.into(),
token: token.into(),
attempts,
@@ -1657,6 +1791,27 @@ mod tests {
}
}
+ #[test]
+ fn timeouts_respect_run_budget_and_eventually_retire_unacknowledged_delivery() {
+ let mut state = WatcherState::default();
+ let mut p = pending("jobs", "12", 0, false);
+ p.since = Instant::now() - Duration::from_secs(700);
+ state.pending.insert("key".into(), p);
+ let config: PipeConfig =
+ serde_json::from_value(serde_json::json!({"schedule": "manual", "timeout": 1200}))
+ .unwrap();
+ let pipes = vec![("jobs".into(), config)];
+ expire_timeouts(&mut state, &pipes);
+ assert!(!state.pending["key"].failed);
+ let p = state.pending.get_mut("key").unwrap();
+ p.since = Instant::now() - Duration::from_secs(1261);
+ p.attempts = RETRY_CAP - 1;
+ expire_timeouts(&mut state, &pipes);
+ assert!(state.pending.is_empty());
+ assert_eq!(state.committed["key"].token, "12");
+ assert!(state.dirty);
+ }
+
#[test]
fn token_cmp_numeric_and_chronological() {
assert_eq!(
@@ -1751,7 +1906,7 @@ mod tests {
let mut s = WatcherState::default();
s.committed.insert("k".into(), committed("0"));
s.pending.insert("k".into(), pending("p", "100", 0, false));
- assert!(apply_completion(&mut s, "p", true));
+ assert!(apply_completion(&mut s, "p", true, Some("test-delivery")));
assert_eq!(s.committed["k"].token, "100");
assert!(!s.pending.contains_key("k"));
}
@@ -1763,12 +1918,12 @@ mod tests {
s.pending.insert("k".into(), pending("p", "100", 0, false));
// fail a few times — stays pending (failed), cursor not advanced
for _ in 0..(RETRY_CAP - 1) {
- apply_completion(&mut s, "p", false);
+ apply_completion(&mut s, "p", false, Some("test-delivery"));
assert!(s.pending.contains_key("k"));
assert_eq!(s.committed["k"].token, "0");
}
// final failure crosses the cap → give up, commit, drop pending
- assert!(apply_completion(&mut s, "p", false));
+ assert!(apply_completion(&mut s, "p", false, Some("test-delivery")));
assert!(!s.pending.contains_key("k"));
assert_eq!(s.committed["k"].token, "100");
}
diff --git a/crates/screenpipe-core/src/pipes/connection_triggers/voice.rs b/crates/screenpipe-core/src/pipes/connection_triggers/voice.rs
new file mode 100644
index 000000000..82549be8b
--- /dev/null
+++ b/crates/screenpipe-core/src/pipes/connection_triggers/voice.rs
@@ -0,0 +1,636 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Spoken phrase detection over newly persisted transcripts. No model calls,
+//! recorder changes, or capture-thread work. Reuses the watcher's durable cursor
+//! and completion/retry protocol; detection runs on its 30-second poll.
+
+use super::{DetectedItem, SourceCtx, SourceTrigger};
+
+pub(super) const MAX_PHRASES: usize = 32;
+pub(super) const MAX_PHRASE_CHARS: usize = 256;
+/// SQL bounds returned text as well as rows. No scan/match on the capture thread.
+const MAX_TRANSCRIPT_CHARS: usize = 16_384;
+/// Bound event-bus payloads too; remaining rows stay behind the cursor.
+const MAX_BATCH_BYTES: usize = 64 * 1024;
+
+pub(super) fn valid_source(src: &SourceTrigger) -> bool {
+ if src.kind != "phrase" && !src.kind.is_empty() {
+ return false;
+ }
+ if !matches!(
+ src.filter.get("device").map(String::as_str),
+ None | Some("input" | "all")
+ ) {
+ return false;
+ }
+ let Some(value) = src.filter.get("phrases") else {
+ return false;
+ };
+ if value.len() > MAX_PHRASES * MAX_PHRASE_CHARS * 4 {
+ return false;
+ }
+ let lines: Vec<_> = value
+ .lines()
+ .filter(|p| !p.trim().is_empty())
+ .take(MAX_PHRASES + 1)
+ .collect();
+ !lines.is_empty()
+ && lines.len() <= MAX_PHRASES
+ && lines
+ .iter()
+ .all(|p| p.chars().count() <= MAX_PHRASE_CHARS && p.chars().any(char::is_alphanumeric))
+}
+
+fn words(text: &str) -> Vec<String> {
+ text.split(|c: char| !c.is_alphanumeric())
+ .filter(|word| !word.is_empty())
+ .map(str::to_lowercase)
+ .collect()
+}
+
+fn phrases(src: &SourceTrigger) -> Vec<String> {
+ src.filter
+ .get("phrases")
+ .into_iter()
+ .flat_map(|value| value.lines())
+ .map(words)
+ .filter(|phrase| !phrase.is_empty())
+ .map(|phrase| format!(" {} ", phrase.join(" ")))
+ .collect()
+}
+
+pub(super) fn matching_items(src: &SourceTrigger, items: Vec<DetectedItem>) -> Vec<DetectedItem> {
+ if !valid_source(src) {
+ return Vec::new();
+ }
+ let phrases = phrases(src);
+ let all_audio = src.filter.get("device").map(String::as_str) == Some("all");
+ items
+ .into_iter()
+ .filter(|item| {
+ if !all_audio && item.is_input != Some(true) {
+ return false;
+ }
+ // Boundary padding preserves whole-word matching. str::contains
+ // uses Rust's substring search instead of quadratic token windows
+ // on repeated-word transcripts and long overlapping phrases.
+ let text = format!(" {} ", words(&item.preview).join(" "));
+ phrases.iter().any(|phrase| text.contains(phrase.as_str()))
+ })
+ .collect()
+}
+
+/// Never turn a truncated final word (e.g. "jobber") into a matching "job".
+fn bounded_preview(text: &str) -> String {
+ let mut chars = text.chars();
+ let mut preview: String = chars.by_ref().take(MAX_TRANSCRIPT_CHARS).collect();
+ if chars.next().is_some_and(char::is_alphanumeric)
+ && preview
+ .chars()
+ .next_back()
+ .is_some_and(char::is_alphanumeric)
+ {
+ let end = preview.rfind(|c: char| !c.is_alphanumeric()).unwrap_or(0);
+ preview.truncate(end);
+ }
+ preview
+}
+
+pub(super) async fn fetch(
+ ctx: &SourceCtx<'_>,
+ src: &SourceTrigger,
+ since: &str,
+) -> Option<Vec<DetectedItem>> {
+ if !valid_source(src) {
+ return None;
+ }
+ // Only fixed SQL and parsed integers are interpolated. User phrases are
+ // matched in Rust, never interpreted as SQL, regex, or instructions.
+ let query = if since.is_empty() {
+ "SELECT COALESCE(MAX(id), 0) AS id FROM audio_transcriptions".to_string()
+ } else {
+ let after = since.parse::<i64>().ok()?.max(0);
+ // Scan both devices by insertion id so system-only activity still
+ // advances a microphone subscription rather than rescanning forever.
+ let read_chars = MAX_TRANSCRIPT_CHARS + 1;
+ format!("SELECT id, timestamp, substr(transcription, 1, {read_chars}) AS transcription, is_input_device FROM audio_transcriptions WHERE id > {after} ORDER BY id ASC LIMIT 50")
+ };
+ // Bound the source's latency independently of remote connection proxies.
+ let response = tokio::time::timeout(
+ std::time::Duration::from_secs(2),
+ ctx.post_json(
+ &format!("{}/raw_sql", ctx.api_base),
+ serde_json::json!({"query": query}),
+ ),
+ )
+ .await
+ .ok()??;
+ let rows = response.as_array()?;
+ let mut items = Vec::with_capacity(rows.len());
+ let mut bytes = 0;
+ for row in rows {
+ let id = row.get("id")?.as_i64()?.to_string();
+ let timestamp = row.get("timestamp").and_then(|v| v.as_str()).unwrap_or("");
+ let is_input = row
+ .get("is_input_device")
+ .is_some_and(|v| v.as_bool() == Some(true) || v.as_i64() == Some(1));
+ let preview = bounded_preview(
+ row.get("transcription")
+ .and_then(|v| v.as_str())
+ .unwrap_or(""),
+ );
+ if bytes + preview.len() > MAX_BATCH_BYTES && !items.is_empty() {
+ break;
+ }
+ bytes += preview.len();
+ items.push(DetectedItem {
+ id: id.clone(),
+ title: timestamp.to_string(),
+ is_input: Some(is_input),
+ preview,
+ ts: id,
+ });
+ }
+ Some(items)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{decide, token_cmp, CursorState, Decision};
+ use super::*;
+
+ fn source(phrases: &str) -> SourceTrigger {
+ serde_json::from_value(serde_json::json!({
+ "app": "audio", "kind": "phrase", "filter": {"phrases": phrases}
+ }))
+ .unwrap()
+ }
+ fn item(id: i64, text: &str) -> DetectedItem {
+ DetectedItem {
+ is_input: Some(true),
+ id: id.to_string(),
+ ts: id.to_string(),
+ title: "2026-09-17T00:15:00Z".into(),
+ preview: text.into(),
+ }
+ }
+
+ #[test]
+ fn voice_matches_case_punctuation_and_job_details_once() {
+ let matched = matching_items(
+ &source("start job\nstop job"),
+ vec![
+ item(
+ 1,
+ "START, job number four, client Acme. Then stop job four.",
+ ),
+ item(2, "restart job four"),
+ item(3, "start a different job"),
+ item(4, "Stop job four."),
+ ],
+ );
+ assert_eq!(
+ matched.iter().map(|i| i.id.as_str()).collect::<Vec<_>>(),
+ vec!["1", "4"]
+ );
+ assert!(matched[0].preview.contains("client Acme"));
+ assert_eq!(matched[0].title, "2026-09-17T00:15:00Z");
+ }
+
+ #[test]
+ fn voice_empty_or_punctuation_only_phrases_never_match() {
+ for phrase in ["", " ", "!!!\n..."] {
+ assert!(matching_items(&source(phrase), vec![item(1, "anything")]).is_empty());
+ }
+ }
+
+ #[test]
+ fn voice_unicode_and_literal_metacharacters() {
+ assert_eq!(
+ matching_items(
+ &source("DÉMARRER tâche"),
+ vec![item(1, "démarrer TÂCHE quatre")]
+ )
+ .len(),
+ 1
+ );
+ assert!(matching_items(
+ &source("start .* job"),
+ vec![item(1, "start unrelated job")]
+ )
+ .is_empty());
+ }
+
+ #[test]
+ fn voice_baselines_without_replay_and_commits_nonmatches() {
+ assert!(
+ matches!(decide("audio", None, None, &[item(90, "start job")], ""), Decision::Init(t) if t == "90")
+ );
+ let cursor = CursorState {
+ initialized: true,
+ token: "90".into(),
+ };
+ let raw = vec![
+ item(90, "start job"),
+ item(91, "ordinary conversation"),
+ item(92, "start job four"),
+ item(93, "ordinary conversation"),
+ ];
+ let Decision::Emit { items, token, .. } = decide("audio", Some(&cursor), None, &raw, "")
+ else {
+ panic!("new rows must be scanned")
+ };
+ assert_eq!(token, "93");
+ assert_eq!(matching_items(&source("start job"), items).len(), 1);
+ let committed = CursorState {
+ initialized: true,
+ token,
+ };
+ assert!(matches!(
+ decide("audio", Some(&committed), None, &raw, ""),
+ Decision::Skip
+ ));
+ assert_eq!(
+ token_cmp("audio", "9007199254740993", "9007199254740992"),
+ std::cmp::Ordering::Greater
+ );
+ }
+ #[tokio::test]
+ async fn voice_fetch_is_authenticated_bounded_and_microphone_only() {
+ use wiremock::matchers::{header, method, path};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+ let server = MockServer::start().await;
+ Mock::given(method("POST")).and(path("/raw_sql"))
+ .and(header("authorization", "Bearer test-local-key"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
+ {"id": 102, "timestamp": "2026-09-17T00:15:00Z", "transcription": "start job four, client Acme", "is_input_device": 1},
+ {"id": 103, "timestamp": "2026-09-17T00:15:01Z", "transcription": "stop job four", "is_input_device": 0}
+ ]))).mount(&server).await;
+ let client = reqwest::Client::new();
+ let ctx = SourceCtx {
+ http: &client,
+ api_base: &server.uri(),
+ api_key: Some("test-local-key"),
+ };
+ let mut src = source("start job'); DROP TABLE audio_transcriptions; --");
+ let rows = fetch(&ctx, &src, "100").await.unwrap();
+ assert_eq!(rows[0].preview, "start job four, client Acme");
+ assert_eq!(rows[0].id, "102");
+ assert_eq!(rows[1].is_input, Some(false));
+ assert!(matching_items(&source("stop job"), rows.clone()).is_empty());
+ assert_eq!(rows[1].ts, "103");
+ src.filter.insert("device".into(), "all".into());
+ assert_eq!(
+ fetch(&ctx, &src, "100").await.unwrap()[1].preview,
+ "stop job four"
+ );
+ fetch(&ctx, &src, "").await.unwrap();
+ assert!(fetch(&ctx, &src, "0; DROP TABLE audio_transcriptions")
+ .await
+ .is_none());
+ let reqs = server.received_requests().await.unwrap();
+ assert_eq!(reqs.len(), 3);
+ let queries: Vec<String> = reqs
+ .iter()
+ .map(|r| {
+ serde_json::from_slice::<serde_json::Value>(&r.body).unwrap()["query"]
+ .as_str()
+ .unwrap()
+ .to_string()
+ })
+ .collect();
+ assert_eq!(queries[0], "SELECT id, timestamp, substr(transcription, 1, 16385) AS transcription, is_input_device FROM audio_transcriptions WHERE id > 100 ORDER BY id ASC LIMIT 50");
+ assert_eq!(queries[0], queries[1]);
+ assert_eq!(
+ queries[2],
+ "SELECT COALESCE(MAX(id), 0) AS id FROM audio_transcriptions"
+ );
+ }
+
+ #[tokio::test]
+ async fn voice_subscriber_skips_idle_audio_retries_and_survives_restart() {
+ use super::super::{apply_completion, process_subscriber, subscription_key, WatcherState};
+ let dir = tempfile::tempdir().unwrap();
+ let pipe = "job-notes";
+ std::fs::create_dir(dir.path().join(pipe)).unwrap();
+ let src = source("start job\nstop job");
+ let key = subscription_key(pipe, &src);
+ let mut state = WatcherState::default();
+ process_subscriber(&mut state, pipe, &src, &key, &[item(90, "start job old")]);
+ assert!(state.pending.is_empty());
+ process_subscriber(
+ &mut state,
+ pipe,
+ &src,
+ &key,
+ &[item(91, "ordinary conversation")],
+ );
+ assert_eq!(state.committed[&key].token, "91");
+ assert!(!dir.path().join(pipe).join(".trigger-context.json").exists());
+ let raw = vec![
+ item(92, "start job four, client Acme"),
+ item(93, "nothing to run"),
+ ];
+ process_subscriber(&mut state, pipe, &src, &key, &raw);
+ assert_eq!(state.pending[&key].token, "93");
+ assert_eq!(state.committed[&key].token, "91");
+ let context = super::super::trigger_context(&src, &matching_items(&src, raw.clone()));
+ let delivery_id = state.pending[&key].delivery_id.clone();
+ assert!(!apply_completion(&mut state, pipe, true, None));
+ assert!(!apply_completion(
+ &mut state,
+ pipe,
+ true,
+ Some("unrelated-run")
+ ));
+ assert_eq!(context["items"].as_array().unwrap().len(), 1);
+ assert_eq!(
+ context["items"][0]["preview"],
+ "start job four, client Acme"
+ );
+ assert!(!apply_completion(
+ &mut state,
+ pipe,
+ false,
+ Some(&delivery_id)
+ ));
+ process_subscriber(&mut state, pipe, &src, &key, &raw);
+ assert_eq!(state.pending[&key].attempts, 1);
+ assert!(apply_completion(&mut state, pipe, true, Some(&delivery_id)));
+ state.save(dir.path());
+ let mut reloaded = WatcherState::load(dir.path());
+ process_subscriber(&mut reloaded, pipe, &src, &key, &raw);
+ assert!(reloaded.pending.is_empty());
+ assert_eq!(reloaded.committed[&key].token, "93");
+ }
+ #[test]
+ fn voice_truncation_cannot_create_a_false_word_match() {
+ let prefix = " ".repeat(MAX_TRANSCRIPT_CHARS - "start job".len());
+ assert!(matching_items(
+ &source("start job"),
+ vec![item(1, &bounded_preview(&format!("{prefix}start jobber")))]
+ )
+ .is_empty());
+ assert_eq!(
+ matching_items(
+ &source("start job"),
+ vec![item(
+ 1,
+ &bounded_preview(&format!("{prefix}start job, more"))
+ )]
+ )
+ .len(),
+ 1
+ );
+ assert_eq!(bounded_preview("start job"), "start job");
+ }
+
+ #[test]
+ fn voice_rejects_invalid_or_unbounded_configuration() {
+ for value in ["x".repeat(257), vec!["start"; 33].join("\n")] {
+ assert!(!valid_source(&source(&value)));
+ }
+ let mut src = source("start job");
+ src.filter.insert("device".into(), "output-typo".into());
+ assert!(!valid_source(&src));
+ assert!(valid_source(&source(&vec!["é".repeat(256); 32].join("\n"))));
+ // A phrase cannot cross segment boundaries or match part of a word.
+ assert!(matching_items(
+ &source("start job"),
+ vec![item(1, "start"), item(2, "job"), item(3, "restart job")]
+ )
+ .is_empty());
+ }
+
+ #[tokio::test]
+ async fn voice_poll_shares_reads_and_retries_an_immutable_snapshot() {
+ use super::super::{poll_once, subscription_key, WatcherState};
+ use wiremock::matchers::{method, path};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/raw_sql"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
+ {"id": 11, "transcription": "start job Acme", "is_input_device": 1},
+ {"id": 12, "transcription": "stop job Acme", "is_input_device": 0}
+ ])))
+ .mount(&server)
+ .await;
+ let mic = source("start job");
+ let mut all = source("stop job");
+ all.filter.insert("device".into(), "all".into());
+ let pipes = vec![(
+ "jobs".into(),
+ serde_json::from_value(serde_json::json!({
+ "schedule": "manual", "enabled": true, "trigger": {"sources": [mic, all]}
+ }))
+ .unwrap(),
+ )];
+ let mut state = WatcherState::default();
+ let mic_key = subscription_key("jobs", &mic);
+ let all_key = subscription_key("jobs", &all);
+ for key in [&mic_key, &all_key] {
+ state.committed.insert(
+ key.clone(),
+ CursorState {
+ token: "10".into(),
+ initialized: true,
+ },
+ );
+ }
+ let client = reqwest::Client::new();
+ let api_base = server.uri();
+ let ctx = SourceCtx {
+ http: &client,
+ api_base: &api_base,
+ api_key: None,
+ };
+ let dir = tempfile::tempdir().unwrap();
+ poll_once(dir.path(), &pipes, &mut state, &ctx, &[]).await;
+ assert_eq!(server.received_requests().await.unwrap().len(), 1);
+ assert_eq!(state.pending.len(), 2);
+ assert_eq!(state.pending[&mic_key].context["items"][0]["id"], "11");
+ assert_eq!(state.pending[&all_key].context["items"][0]["id"], "12");
+ let original = state.pending[&mic_key].clone();
+ poll_once(dir.path(), &pipes, &mut state, &ctx, &[]).await;
+ assert_eq!(
+ server.received_requests().await.unwrap().len(),
+ 1,
+ "no reads while pending"
+ );
+ poll_once(
+ dir.path(),
+ &pipes,
+ &mut state,
+ &ctx,
+ &[("jobs".into(), false, Some(original.delivery_id.clone()))],
+ )
+ .await;
+ assert_eq!(
+ server.received_requests().await.unwrap().len(),
+ 1,
+ "retries reuse their snapshot"
+ );
+ assert_eq!(state.pending[&mic_key].context, original.context);
+ assert_eq!(state.pending[&mic_key].delivery_id, original.delivery_id);
+ assert_eq!(state.pending[&mic_key].attempts, 1);
+ assert_eq!(state.pending[&all_key].attempts, 0);
+ assert!(super::super::apply_completion(
+ &mut state,
+ "jobs",
+ true,
+ Some(&original.delivery_id)
+ ));
+ assert!(
+ state.pending.contains_key(&all_key),
+ "one completion cannot acknowledge another source"
+ );
+ assert_eq!(state.committed[&all_key].token, "10");
+ }
+
+ #[tokio::test]
+ async fn voice_batch_budget_leaves_excess_rows_for_next_poll() {
+ use wiremock::matchers::method;
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+ let server = MockServer::start().await;
+ let rows: Vec<_> = (1..=6).map(|id| serde_json::json!({"id": id, "transcription": "x".repeat(MAX_TRANSCRIPT_CHARS), "is_input_device": 1})).collect();
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(rows))
+ .mount(&server)
+ .await;
+ let client = reqwest::Client::new();
+ let api_base = server.uri();
+ let ctx = SourceCtx {
+ http: &client,
+ api_base: &api_base,
+ api_key: None,
+ };
+ let batch = fetch(&ctx, &source("start job"), "0").await.unwrap();
+ assert_eq!(batch.len(), 4);
+ assert_eq!(batch.last().unwrap().ts, "4");
+ assert_eq!(
+ batch.iter().map(|item| item.preview.len()).sum::<usize>(),
+ MAX_BATCH_BYTES
+ );
+ }
+
+ #[tokio::test]
+ async fn voice_invalid_disabled_and_timed_out_sources_do_not_advance() {
+ use super::super::{poll_once, subscription_key, WatcherState};
+ use wiremock::matchers::method;
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .set_delay(std::time::Duration::from_secs(3))
+ .set_body_json(serde_json::json!([{"id": 99}])),
+ )
+ .mount(&server)
+ .await;
+ let client = reqwest::Client::new();
+ let api_base = server.uri();
+ let ctx = SourceCtx {
+ http: &client,
+ api_base: &api_base,
+ api_key: None,
+ };
+ let dir = tempfile::tempdir().unwrap();
+ let mut state = WatcherState::default();
+ let config = |enabled, src: &SourceTrigger| {
+ serde_json::from_value(serde_json::json!({"schedule": "manual", "enabled": enabled, "trigger": {"sources": [src]}})).unwrap()
+ };
+ let src = source("start job");
+ poll_once(
+ dir.path(),
+ &[
+ ("jobs".into(), config(false, &src)),
+ ("invalid".into(), config(true, &source("!!!"))),
+ ],
+ &mut state,
+ &ctx,
+ &[],
+ )
+ .await;
+ assert!(server.received_requests().await.unwrap().is_empty());
+ let key = subscription_key("jobs", &src);
+ state.committed.insert(
+ key.clone(),
+ CursorState {
+ token: "10".into(),
+ initialized: true,
+ },
+ );
+ poll_once(
+ dir.path(),
+ &[("jobs".into(), config(true, &src))],
+ &mut state,
+ &ctx,
+ &[],
+ )
+ .await;
+ assert_eq!(state.committed[&key].token, "10");
+ assert!(state.pending.is_empty());
+ }
+
+ #[tokio::test]
+ async fn voice_new_subscription_baselines_and_read_failure_preserves_cursor() {
+ use super::super::{poll_once, subscription_key, WatcherState};
+ use wiremock::matchers::{body_json, method};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(
+ serde_json::json!([{"id": 11, "transcription": "start job", "is_input_device": 1}]),
+ ))
+ .with_priority(2)
+ .mount(&server)
+ .await;
+ Mock::given(body_json(serde_json::json!({"query": "SELECT COALESCE(MAX(id), 0) AS id FROM audio_transcriptions"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([{"id": 1000}]))).with_priority(1).mount(&server).await;
+ let old = source("ordinary phrase");
+ let new = source("start job");
+ let pipes = vec![(
+ "jobs".into(),
+ serde_json::from_value(serde_json::json!({
+ "schedule": "manual", "trigger": {"sources": [old, new]}
+ }))
+ .unwrap(),
+ )];
+ let mut state = WatcherState::default();
+ let old_key = subscription_key("jobs", &old);
+ let new_key = subscription_key("jobs", &new);
+ state.committed.insert(
+ old_key.clone(),
+ CursorState {
+ token: "10".into(),
+ initialized: true,
+ },
+ );
+ let client = reqwest::Client::new();
+ let api_base = server.uri();
+ let ctx = SourceCtx {
+ http: &client,
+ api_base: &api_base,
+ api_key: None,
+ };
+ let dir = tempfile::tempdir().unwrap();
+ poll_once(dir.path(), &pipes, &mut state, &ctx, &[]).await;
+ assert_eq!(
+ state.committed[&new_key].token, "1000",
+ "new source must not replay the old subscriber's backlog"
+ );
+ assert_eq!(state.committed[&old_key].token, "11");
+ assert!(state.pending.is_empty());
+ assert_eq!(server.received_requests().await.unwrap().len(), 2);
+ server.reset().await;
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(503))
+ .mount(&server)
+ .await;
+ poll_once(dir.path(), &pipes, &mut state, &ctx, &[]).await;
+ assert_eq!(state.committed[&old_key].token, "11");
+ assert_eq!(state.committed[&new_key].token, "1000");
+ }
+}
diff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rs
index 89d240e09..b0d979dc1 100644
--- a/crates/screenpipe-core/src/pipes/mod.rs
+++ b/crates/screenpipe-core/src/pipes/mod.rs
@@ -259,8 +259,34 @@ fn is_empty_str_map(m: &std::collections::HashMap<String, String>) -> bool {
/// path: /Users/me/vault/meetings
/// ```
///
-/// The watcher writes the new items to `<pipe-dir>/.trigger-context.json` before
-/// firing, so the pipe prompt can read exactly what changed (cwd is the pipe dir).
+/// Spoken phrases use the same source contract without an external connection:
+/// ```yaml
+/// schedule: manual
+/// trigger:
+/// sources:
+/// - app: audio
+/// kind: phrase
+/// filter:
+/// phrases: |-
+/// start job
+/// stop job
+/// device: input
+/// ```
+/// Voice matching checks newly saved transcript segments every 30 seconds,
+/// ignoring case and punctuation and requiring whole consecutive words. It
+/// does not stitch phrases across segments or identify a particular speaker.
+/// `device: input` (default) watches microphones; `all` includes system audio.
+/// No model runs until a phrase matches. Transcription must already be enabled.
+/// Voice items use the transcription row id for `id` and the cursor `ts`, the
+/// capture timestamp for `title`, and up to 16,384 transcript characters for `preview`.
+/// Matching is limited to this prefix of each segment, without partial words.
+/// Batches carry at most 64 KiB of transcript text; excess rows wait for the next poll.
+/// At most 32 phrases of 256 characters each are accepted per source.
+/// Keep related phrases in one source so one run receives all matching items.
+///
+/// The watcher carries matching items in the existing ConnectionTriggerEvent.
+/// The scheduler writes `<pipe-dir>/.trigger-context.json` only when the pipe
+/// is idle and selected to run, then echoes the delivery id on completion.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SourceTrigger {
/// Connected app id, e.g. "obsidian". Apps the watcher can't poll yet are
@@ -6150,33 +6176,17 @@ impl PipeManager {
}
}
- // connection_trigger events are addressed to a specific pipe
- // (the watcher already matched the source and wrote
- // .trigger-context.json), so fire that pipe directly — no
- // trigger.events string matching needed.
+ // Source deliveries use the same matching/defer path as
+ // meeting events. Context travels with the event instead of
+ // being written by a producer while a prior run is active.
while let Some(e) = connection_trigger_rx.next().now_or_never().flatten() {
- if let Some(target) = e.data.get("pipe").and_then(|v| v.as_str()) {
- for (name, config, _body) in &pipe_snapshot {
- if name == target && config.enabled {
- info!("scheduler: connection trigger fired pipe '{}'", name);
- last_run.remove(name);
- // No key: connection triggers already have their
- // own committed-cursor dedupe in
- // connection_triggers.rs, so they must not be
- // suppressed a second time here.
- event_triggered.insert(
- name.clone(),
- EventTrigger {
- name: e.name.clone(),
- key: None,
- dedupe_key: None,
- target_pipe: None,
- data: e.data.clone(),
- },
- );
- connection_triggered.insert(name.clone());
- }
- }
+ if let Some(target) = e
+ .data
+ .get("pipe")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned)
+ {
+ pending_events.push(PendingEvent::targeted(e.name, e.data, &target));
}
}
@@ -6194,7 +6204,7 @@ impl PipeManager {
continue;
}
if let Some(ref trigger) = config.trigger {
- if !trigger.events.iter().any(|e| e == event_name) {
+ if !pending_matches_trigger(pending, name, trigger) {
continue;
}
@@ -6222,8 +6232,21 @@ impl PipeManager {
let incoming_trigger = EventTrigger {
name: event_name.clone(),
- key: event_identity_key(data),
- dedupe_key: event_dedupe_key(event_name, data),
+ key: if event_name == "connection_trigger" {
+ data.get("delivery_id")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned)
+ } else {
+ event_identity_key(data)
+ },
+ // Source cursors own their acknowledgment/retry
+ // protocol; a scheduler DB claim would prevent
+ // redelivery after a lost completion event.
+ dedupe_key: if event_name == "connection_trigger" {
+ None
+ } else {
+ event_dedupe_key(event_name, data)
+ },
target_pipe: pending.target_pipe.clone(),
data: data.clone(),
};
@@ -6285,6 +6308,13 @@ impl PipeManager {
}
}
+ connection_triggered.extend(
+ event_triggered
+ .iter()
+ .filter(|(_, event)| event.name == "connection_trigger")
+ .map(|(name, _)| name.clone()),
+ );
+
for (name, config, body) in &pipe_snapshot {
if !config.enabled {
continue;
@@ -6345,18 +6375,12 @@ impl PipeManager {
if qr.contains(name) {
// Hold an event trigger back rather than dropping it —
// the run in flight may be for a different event.
- // Connection triggers are excluded: they are addressed
- // to a pipe by name, so replaying them through event
- // matching wouldn't fire anything, and their watcher
- // already retries uncommitted fires.
- if !connection_triggered.contains(name) {
- if let Some(t) = event_triggered.get(name) {
- deferred.push(PendingEvent {
- name: t.name.clone(),
- data: t.data.clone(),
- target_pipe: t.target_pipe.clone(),
- });
- }
+ if let Some(t) = event_triggered.get(name) {
+ deferred.push(PendingEvent {
+ name: t.name.clone(),
+ data: t.data.clone(),
+ target_pipe: t.target_pipe.clone(),
+ });
}
continue;
}
@@ -6760,17 +6784,31 @@ impl PipeManager {
let queued_ref = queued_or_running.clone();
let mcp_server_allowlist = selected_mcp_server_ids(config);
- // Tell the pipe which event fired it, so it acts on that
- // meeting instead of guessing at the most recent one.
- //
- // Skipped for connection triggers: their watcher already wrote
- // the same file with the items it detected, and overwriting it
- // would take that payload away from the pipe.
- if !connection_triggered.contains(name) {
- if let Some(t) = event_triggered.get(name) {
- write_event_trigger_context(&pipe_dir, t);
+ // This pipe has passed the busy/queue guards. Only its
+ // selected delivery can now replace the context file.
+ if let Some(t) = event_triggered.get(name) {
+ if !write_event_trigger_context(&pipe_dir, t)
+ && t.name == "connection_trigger"
+ {
+ warn!(
+ "scheduler: cannot persist source context for '{}'; deferring",
+ name
+ );
+ queued_or_running.lock().await.remove(name);
+ deferred.push(PendingEvent {
+ name: t.name.clone(),
+ data: t.data.clone(),
+ target_pipe: t.target_pipe.clone(),
+ });
+ continue;
}
}
+ let source_delivery_id = event_triggered
+ .get(name)
+ .filter(|t| t.name == "connection_trigger")
+ .and_then(|t| t.data.get("delivery_id"))
+ .and_then(|v| v.as_str())
+ .map(str::to_owned);
let claim_for_release = event_claim.clone();
// Execution metadata and the pipe-facing context keep the
@@ -7258,7 +7296,12 @@ impl PipeManager {
}
// Emit pipe_completed event so other pipes can chain
- emit_pipe_completed(&name_for_cb, success, duration_secs);
+ emit_pipe_completed_with_delivery(
+ &name_for_cb,
+ success,
+ duration_secs,
+ source_delivery_id,
+ );
// Fire run-complete callback (analytics, etc.)
if let Some(ref cb) = on_complete {
@@ -7415,7 +7458,7 @@ impl PipeManager {
.collect()
};
// Drain pipe_completed:* events since the last tick.
- let mut completions: Vec<(String, bool)> = Vec::new();
+ let mut completions: Vec<(String, bool, Option<String>)> = Vec::new();
while let Some(e) = completed_rx.next().now_or_never().flatten() {
if e.name.starts_with("pipe_completed:") {
let pipe = e
@@ -7429,7 +7472,14 @@ impl PipeManager {
.and_then(|v| v.as_bool())
.unwrap_or(false);
if let Some(pipe) = pipe {
- completions.push((pipe, success));
+ completions.push((
+ pipe,
+ success,
+ e.data
+ .get("source_delivery_id")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned),
+ ));
}
}
}
@@ -7836,19 +7886,27 @@ const EVENT_TRIGGER_CONTEXT_FILE: &str = ".trigger-context.json";
///
/// Without this a `meeting_ended` pipe has to guess which meeting it was woken
/// for, which goes wrong exactly when two meetings end close together.
-fn write_event_trigger_context(pipe_dir: &Path, trigger: &EventTrigger) {
+fn write_event_trigger_context(pipe_dir: &Path, trigger: &EventTrigger) -> bool {
if !pipe_dir.is_dir() {
- return;
- }
- let ctx = serde_json::json!({
- "event": trigger.name,
- "key": trigger.key,
- "data": trigger.data,
- "triggered_at": Utc::now().to_rfc3339(),
- });
- if let Ok(s) = serde_json::to_string_pretty(&ctx) {
- let _ = atomic_write(&pipe_dir.join(EVENT_TRIGGER_CONTEXT_FILE), &s);
+ return false;
}
+ let ctx = if trigger.name == "connection_trigger" {
+ // Preserve the existing source context shape for installed Pipe prompts.
+ match trigger.data.get("context") {
+ Some(context) if context.is_object() => context.clone(),
+ _ => return trigger.data.get("delivery_id").is_none(), // legacy producers wrote their own context
+ }
+ } else {
+ serde_json::json!({
+ "event": trigger.name,
+ "key": trigger.key,
+ "data": trigger.data,
+ "triggered_at": Utc::now().to_rfc3339(),
+ })
+ };
+ serde_json::to_string_pretty(&ctx)
+ .ok()
+ .is_some_and(|s| atomic_write(&pipe_dir.join(EVENT_TRIGGER_CONTEXT_FILE), &s).is_ok())
}
/// One event delivery that matched a pipe's `trigger.events`.
@@ -7920,6 +7978,9 @@ fn resolve_same_tick_event(
return SameTickEventResolution::DeferIncoming;
}
+ if existing.name == "connection_trigger" && existing.key == incoming.key {
+ return SameTickEventResolution::KeepExisting;
+ }
if existing.dedupe_key.is_some() && existing.dedupe_key == incoming.dedupe_key {
return SameTickEventResolution::KeepExisting;
}
@@ -7968,6 +8029,28 @@ impl PendingEvent {
}
}
+/// Addressed source deliveries share the scheduler's ordinary event queue,
+/// but their subscription must still exist when the delayed event is consumed.
+fn pending_matches_trigger(pending: &PendingEvent, pipe: &str, trigger: &TriggerConfig) -> bool {
+ if pending.name != "connection_trigger" {
+ return trigger.events.iter().any(|event| event == &pending.name);
+ }
+ if pending.target_pipe.as_deref() != Some(pipe) {
+ return false;
+ }
+ match pending
+ .data
+ .get("subscription_key")
+ .and_then(|v| v.as_str())
+ {
+ Some(key) => trigger
+ .sources
+ .iter()
+ .any(|source| connection_triggers::subscription_key(pipe, source) == key),
+ None => !trigger.sources.is_empty(), // legacy source producer
+ }
+}
+
/// Emitters set `skip_pipes` on events describing something too trivial to be
/// worth an agent run (a seconds-long meeting with no transcript). Lifecycle
/// consumers on the same topic ignore the field.
@@ -8015,6 +8098,13 @@ fn event_identity_key(data: &serde_json::Value) -> Option<String> {
/// new transcript generation that needs its own summary. Legacy emitters that
/// do not include `meeting_end` retain the old meeting-id-only behavior.
fn event_dedupe_key(event_name: &str, data: &serde_json::Value) -> Option<String> {
+ if event_name == "connection_trigger" {
+ return data
+ .get("delivery_id")
+ .and_then(|v| v.as_str())
+ .filter(|id| !id.is_empty())
+ .map(str::to_owned);
+ }
let identity = event_identity_key(data)?;
if event_name != "meeting_ended" {
return Some(identity);
@@ -8033,13 +8123,23 @@ fn event_dedupe_key(event_name: &str, data: &serde_json::Value) -> Option<String
}
fn emit_pipe_completed(pipe_name: &str, success: bool, duration_secs: f64) {
+ emit_pipe_completed_with_delivery(pipe_name, success, duration_secs, None);
+}
+
+fn emit_pipe_completed_with_delivery(
+ pipe_name: &str,
+ success: bool,
+ duration_secs: f64,
+ source_delivery_id: Option<String>,
+) {
let _ = screenpipe_events::send_event(
- &format!("{PIPE_COMPLETED_EVENT_PREFIX}{pipe_name}"),
+ format!("{PIPE_COMPLETED_EVENT_PREFIX}{pipe_name}"),
screenpipe_events::PipeCompletedEvent {
pipe_name: pipe_name.to_string(),
success,
duration_secs,
timestamp: Utc::now(),
+ source_delivery_id,
},
);
}
@@ -9111,6 +9211,184 @@ mod tests {
}
}
+ struct SourceContextExecutor {
+ contexts: std::sync::Mutex<Vec<serde_json::Value>>,
+ release: tokio::sync::Semaphore,
+ }
+
+ #[async_trait::async_trait]
+ impl AgentExecutor for SourceContextExecutor {
+ async fn run(
+ &self,
+ _prompt: &str,
+ _model: &str,
+ working_dir: &Path,
+ _provider: Option<&str>,
+ _provider_url: Option<&str>,
+ _provider_api_key: Option<&str>,
+ _shared_pid: Option<SharedPid>,
+ _continue_session: bool,
+ ) -> Result<AgentOutput> {
+ let path = working_dir.join(EVENT_TRIGGER_CONTEXT_FILE);
+ let before = std::fs::read_to_string(&path)?;
+ self.contexts
+ .lock()
+ .unwrap()
+ .push(serde_json::from_str(&before)?);
+ self.release.acquire().await?.forget();
+ assert_eq!(
+ std::fs::read_to_string(&path)?,
+ before,
+ "queued source overwrote active context"
+ );
+ Ok(AgentOutput {
+ stdout: "saved".into(),
+ stderr: String::new(),
+ success: true,
+ pid: None,
+ })
+ }
+ fn kill(&self, _handle: &ExecutionHandle) -> Result<()> {
+ Ok(())
+ }
+ fn is_available(&self) -> bool {
+ true
+ }
+ async fn ensure_installed(&self) -> Result<()> {
+ Ok(())
+ }
+ fn name(&self) -> &str {
+ "source-context-test"
+ }
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn source_events_serialize_context_and_acknowledge_their_own_delivery() {
+ let temp = tempfile::tempdir().unwrap();
+ let pipes_dir = temp.path().join("pipes");
+ let name = "source-context-serialization";
+ let dir = pipes_dir.join(name);
+ std::fs::create_dir_all(&dir).unwrap();
+ std::fs::write(dir.join("pipe.md"), "---\nschedule: manual\nagent: mock\nmodel: test\ntrigger:\n sources:\n - app: audio\n kind: phrase\n filter:\n phrases: start job\n---\nSave the matched transcript.\n").unwrap();
+ let src: SourceTrigger = serde_json::from_value(serde_json::json!({"app": "audio", "kind": "phrase", "filter": {"phrases": "start job"}})).unwrap();
+ let key = connection_triggers::subscription_key(name, &src);
+ let executor = Arc::new(SourceContextExecutor {
+ contexts: std::sync::Mutex::new(Vec::new()),
+ release: tokio::sync::Semaphore::new(0),
+ });
+ let mut executors: HashMap<String, Arc<dyn AgentExecutor>> = HashMap::new();
+ executors.insert("mock".into(), executor.clone());
+ let mut manager = PipeManager::new(pipes_dir, executors, None, 0);
+ manager.load_pipes().await.unwrap();
+ let mut completed = screenpipe_events::subscribe_to_event::<serde_json::Value>(&format!(
+ "pipe_completed:{name}"
+ ));
+ manager.start_scheduler().await.unwrap();
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ let send = |id: &str| {
+ screenpipe_events::send_event(
+ "connection_trigger",
+ screenpipe_events::ConnectionTriggerEvent {
+ pipe: name.into(),
+ app: "audio".into(),
+ kind: "phrase".into(),
+ path: None,
+ count: 1,
+ timestamp: Utc::now(),
+ delivery_id: Some(id.into()),
+ subscription_key: Some(key.clone()),
+ context: Some(serde_json::json!({"app": "audio", "items": [{"preview": id}]})),
+ },
+ )
+ .unwrap();
+ };
+ send("first");
+ tokio::time::advance(std::time::Duration::from_secs(31)).await;
+ for _ in 0..100 {
+ if executor.contexts.lock().unwrap().len() == 1 {
+ break;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+ }
+ assert_eq!(executor.contexts.lock().unwrap().len(), 1);
+ send("second");
+ send("second"); // repeated bus delivery must remain one queued run
+ tokio::time::advance(std::time::Duration::from_secs(31)).await;
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(executor.contexts.lock().unwrap().len(), 1);
+ assert_eq!(
+ serde_json::from_str::<serde_json::Value>(
+ &std::fs::read_to_string(dir.join(EVENT_TRIGGER_CONTEXT_FILE)).unwrap()
+ )
+ .unwrap()["items"][0]["preview"],
+ "first"
+ );
+ executor.release.add_permits(1);
+ let first = tokio::time::timeout(std::time::Duration::from_secs(5), completed.next())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(first.data["source_delivery_id"], "first");
+ assert_eq!(first.data["success"], true);
+ tokio::time::advance(std::time::Duration::from_secs(31)).await;
+ for _ in 0..100 {
+ if executor.contexts.lock().unwrap().len() == 2 {
+ break;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+ }
+ assert_eq!(executor.contexts.lock().unwrap().len(), 2);
+ executor.release.add_permits(1);
+ let second = tokio::time::timeout(std::time::Duration::from_secs(5), completed.next())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(second.data["source_delivery_id"], "second");
+ tokio::time::advance(std::time::Duration::from_secs(61)).await;
+ for _ in 0..20 {
+ tokio::task::yield_now().await;
+ }
+ manager.stop_scheduler().await;
+ assert_eq!(executor.contexts.lock().unwrap().len(), 2);
+ assert_eq!(
+ executor.contexts.lock().unwrap()[1]["items"][0]["preview"],
+ "second"
+ );
+ }
+
+ #[test]
+ fn source_events_validate_subscription_and_fail_closed_without_context() {
+ let src: SourceTrigger = serde_json::from_value(
+ serde_json::json!({"app": "audio", "filter": {"phrases": "start"}}),
+ )
+ .unwrap();
+ let trigger: TriggerConfig =
+ serde_json::from_value(serde_json::json!({"sources": [src]})).unwrap();
+ let data = serde_json::json!({"subscription_key": connection_triggers::subscription_key("jobs", &src), "delivery_id": "one"});
+ let pending = PendingEvent::targeted("connection_trigger".into(), data.clone(), "jobs");
+ assert!(pending_matches_trigger(&pending, "jobs", &trigger));
+ assert!(!pending_matches_trigger(&pending, "another-task", &trigger));
+ assert!(!pending_matches_trigger(
+ &pending,
+ "jobs",
+ &serde_json::from_value(serde_json::json!({})).unwrap()
+ ));
+ let event = EventTrigger {
+ name: "connection_trigger".into(),
+ key: Some("one".into()),
+ dedupe_key: None,
+ target_pipe: Some("jobs".into()),
+ data,
+ };
+ let dir = tempfile::tempdir().unwrap();
+ assert!(!write_event_trigger_context(dir.path(), &event));
+ assert!(!dir.path().join(EVENT_TRIGGER_CONTEXT_FILE).exists());
+ }
+
#[tokio::test(start_paused = true)]
async fn manual_completion_starts_dependent_task_once() {
let temp = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-events/src/custom_events/connections.rs b/crates/screenpipe-events/src/custom_events/connections.rs
index 16183c960..d22c1666f 100644
--- a/crates/screenpipe-events/src/custom_events/connections.rs
+++ b/crates/screenpipe-events/src/custom_events/connections.rs
@@ -30,6 +30,16 @@ pub struct ConnectionTriggerEvent {
pub count: usize,
/// When the batch was detected.
pub timestamp: DateTime<Utc>,
+ /// Identifies this delivery, echoed only by the run it actually started.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub delivery_id: Option<String>,
+ /// Used to reject deliveries after their source configuration is removed.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub subscription_key: Option<String>,
+ /// Immutable per-delivery context. The scheduler writes it only after the
+ /// pipe is idle, so another source cannot replace an active run's input.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub context: Option<serde_json::Value>,
}
#[cfg(test)]
@@ -45,10 +55,27 @@ mod tests {
path: Some("/Users/me/vault/meetings".to_string()),
count: 3,
timestamp: Utc::now(),
+ delivery_id: None,
+ subscription_key: None,
+ context: None,
};
let json = serde_json::to_string(&event).unwrap();
let parsed: ConnectionTriggerEvent = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.pipe, "meeting-notes");
assert_eq!(parsed.count, 3);
}
+ #[test]
+ fn legacy_and_delivery_payloads_are_compatible() {
+ let legacy = serde_json::json!({"pipe": "jobs", "app": "audio", "kind": "phrase", "count": 1, "timestamp": Utc::now()});
+ let mut event: ConnectionTriggerEvent = serde_json::from_value(legacy.clone()).unwrap();
+ assert!(event.delivery_id.is_none());
+ assert_eq!(serde_json::to_value(&event).unwrap(), legacy);
+ event.delivery_id = Some("delivery".into());
+ event.subscription_key = Some("subscription".into());
+ event.context = Some(serde_json::json!({"items": [{"preview": "start job"}]}));
+ let parsed: ConnectionTriggerEvent =
+ serde_json::from_value(serde_json::to_value(&event).unwrap()).unwrap();
+ assert_eq!(parsed.delivery_id, event.delivery_id);
+ assert_eq!(parsed.context, event.context);
+ }
}
diff --git a/crates/screenpipe-events/src/custom_events/pipes.rs b/crates/screenpipe-events/src/custom_events/pipes.rs
index f50c9bf6d..820664b60 100644
--- a/crates/screenpipe-events/src/custom_events/pipes.rs
+++ b/crates/screenpipe-events/src/custom_events/pipes.rs
@@ -18,4 +18,25 @@ pub struct PipeCompletedEvent {
pub success: bool,
pub duration_secs: f64,
pub timestamp: DateTime<Utc>,
+ /// Present only when this execution was started by a source delivery.
+ /// Manual, scheduled, and unrelated event runs cannot acknowledge it.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub source_delivery_id: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn completions_without_a_source_delivery_remain_compatible() {
+ let legacy = serde_json::json!({"pipe_name": "jobs", "success": true, "duration_secs": 1.0, "timestamp": Utc::now()});
+ let mut event: PipeCompletedEvent = serde_json::from_value(legacy.clone()).unwrap();
+ assert!(event.source_delivery_id.is_none());
+ assert_eq!(serde_json::to_value(&event).unwrap(), legacy);
+ event.source_delivery_id = Some("delivery".into());
+ let parsed: PipeCompletedEvent =
+ serde_json::from_value(serde_json::to_value(&event).unwrap()).unwrap();
+ assert_eq!(parsed.source_delivery_id, event.source_delivery_id);
+ }
}