The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(omp): resend a skill listing until the hook reports it recorded
Kyle Elliott35dfb16c0f77
diff --git a/cli/beacon-hooks/cmd/omp_event_test.go b/cli/beacon-hooks/cmd/omp_event_test.goindex 1930a569..61a0af5b 100644--- a/cli/beacon-hooks/cmd/omp_event_test.go+++ b/cli/beacon-hooks/cmd/omp_event_test.go@@ -1,6 +1,7 @@package cmdimport (+ "os""path/filepath""reflect""strings"@@ -248,6 +249,45 @@ func TestOmpEventSkillListingIsWrittenAsSystemContext(t *testing.T) {}}+// The extension decides whether to resend a skill listing from this reply, because the hook exits 0+// whether or not it wrote anything. Every way an event is dropped must report nothing recorded.+func TestOmpEventReplyReportsWhatWasRecorded(t *testing.T) {+ listing := map[string]interface{}{"type": "context", "sessionId": "sess-1", "skillListing": "- deploy: Deploy applications."}+ noLog := func(t *testing.T) {+ for _, key := range []string{"BEACON_ENDPOINT_LOG", "BEACON_CLOUD_LOG_PATH", "BEACON_LOG_PATH", "BEACON_RUNTIME_LOG", "BEACON_ENDPOINT_MODE"} {+ t.Setenv(key, "")+ }+ }+ unwritable := func(t *testing.T) {+ dir := filepath.Join(t.TempDir(), "runtime.jsonl")+ if err := os.Mkdir(dir, 0o755); err != nil {+ t.Fatal(err)+ }+ t.Setenv("BEACON_ENDPOINT_LOG", dir)+ }+ for _, tc := range []struct {+ name string+ input map[string]interface{}+ setup func(*testing.T)+ written float64+ }{+ {"written", listing, nil, 1},+ {"log not writable", listing, unwritable, 0},+ {"no endpoint log configured", listing, noLog, 0},+ {"type this build does not map", map[string]interface{}{"type": "message_update", "sessionId": "sess-1"}, nil, 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ ompTestLog(t)+ if tc.setup != nil {+ tc.setup(t)+ }+ if out := runHookWithInput(t, runOmpEvent, tc.input); out["recorded"] != tc.written {+ t.Fatalf("reply = %v, want recorded = %v", out, tc.written)+ }+ })+ }+}+// Print and ACP never emit `input`; the extension forwards the delivered user message instead,// with its text under `prompt`. The row names that origin, because the text is what Oh My Pi sent// the model after expanding templates and commands, not what an `input` event would carry.diff --git a/cli/beacon-hooks/cmd/pi_event.go b/cli/beacon-hooks/cmd/pi_event.goindex bffac5d5..cc4de2c5 100644--- a/cli/beacon-hooks/cmd/pi_event.go+++ b/cli/beacon-hooks/cmd/pi_event.go@@ -2,6 +2,8 @@ package cmdimport ("github.com/spf13/cobra"++ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging")// pi-event is the single entry point for every Pi lifecycle payload.@@ -32,21 +34,30 @@ func runPiEvent(cmd *cobra.Command, args []string) {//// Shared by pi-event and omp-event because the transport is identical -- the extension spawns the// hook binary and writes one JSON object to its stdin -- and only the runtime it describes differs.+//+// The reply, {"recorded": n}, is how many events reached the log, a duplicate the log already holds+// included. It is the only way the extension can learn that: the exit status is 0 whether or not+// anything was written, because a hook that fails must not fail the run it observes, and an+// envelope that did not parse, a log that cannot be written, a log that is not configured and an+// event type this build does not map all end the same way.func runPiFamilyEvent(runtime piFamily) {input, err := readStdinJSON()if err != nil {- outputJSON(emptyResponse)+ outputJSON(map[string]interface{}{"recorded": 0})return}sessionID := resolveSessionID(input, runtime.platform)logger := newHookLogger(runtime.platform+"-event", runtime.platform, sessionID)+ recorded := 0for _, event := range runtime.endpointEvents(input, sessionID) {if event.action == "" {continue}- _ = logger.EndpointEvent(event.action, event.category, event.severity, event.message, event.fields)+ if logger.EndpointEvent(event.action, event.category, event.severity, event.message, event.fields) == nil && logging.EndpointLogConfigured() {+ recorded+++ }}- outputJSON(emptyResponse)+ outputJSON(map[string]interface{}{"recorded": recorded})}// supportedPiEventTypes lists every Pi event type this mapper handles.diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.goindex 1bef5977..9d2e70e5 100644--- a/cli/beacon-hooks/internal/logging/logging.go+++ b/cli/beacon-hooks/internal/logging/logging.go@@ -514,6 +514,13 @@ func minimalEndpointEvent(event map[string]interface{}) map[string]interface{} {return out}+// EndpointLogConfigured reports whether this process has a runtime log to write to. Without one,+// EndpointEvent records nothing and returns nil, so a caller reporting what it recorded checks this+// rather than reading a nil error as a write.+func EndpointLogConfigured() bool {+ return endpointLogPath() != ""+}+func endpointLogPath() string {if path := firstEnv("BEACON_ENDPOINT_LOG", "BEACON_CLOUD_LOG_PATH", "BEACON_LOG_PATH", "BEACON_RUNTIME_LOG"); path != "" {return pathdiff --git a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.tsindex fbac406e..c02dec5e 100644--- a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts+++ b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts@@ -94,6 +94,18 @@ const skillsSection = /^# Skills & Rules\n(?:[^\n]*\n){0,3}?<skills>\s*([\s\S]*?// How many sessions' last-sent listings are remembered, least recently sent evicted first.const maxRememberedSkillSessions = 64+// A listing the hook did not record is sent again on later model calls, up to this many attempts+// in all. The cap keeps a hook that can never record it -- no endpoint log configured, a hook binary+// older than this extension -- from costing a spawn before every model call.+const maxSkillListingAttempts = 3++interface RememberedSkillListing {+ listing: string+ attempts: number+ // Recorded, or given up on after maxSkillListingAttempts.+ done: boolean+}+function skillListing(ctx: OmpContext | undefined): string {try {const prompt = ctx?.getSystemPrompt?.()@@ -110,7 +122,7 @@ function skillListing(ctx: OmpContext | undefined): string {return ""}-function rememberSkillListing(seen: Map<string, string>, key: string, listing: string) {+function rememberSkillListing(seen: Map<string, RememberedSkillListing>, key: string, listing: RememberedSkillListing) {// Re-inserted rather than updated so the map's insertion order is send order, and eviction// drops the session that has gone longest without a change.seen.delete(key)@@ -142,7 +154,7 @@ const maxPendingToolCalls = 64// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,// not a timestamp or a guess.const pendingToolCalls = new Map<string, unknown>()-const rememberedSkillListings = new Map<string, string>()+const rememberedSkillListings = new Map<string, RememberedSkillListing>()function rememberToolCall(event: OmpEvent) {const id = typeof event.toolCallId === "string" ? event.toolCallId : ""@@ -241,9 +253,26 @@ function safeClone(value: unknown, depth = 0, seen = new WeakSet<object>()): unk}}-// sendToBeacon resolves true only when the hook binary read the event and exited cleanly. Every-// failure is still swallowed -- telemetry never interrupts the run -- but a caller that sends-// something once rather than per event needs to know whether to try again.+// The most of the hook binary's reply that is read. The reply is one small object, {"recorded": n}.+const maxReplyBytes = 4096++// hookRecorded reads the hook binary's reply: whether it wrote at least one event. The exit status+// cannot say, because the hook exits 0 whether or not it wrote anything -- a hook that failed must+// not fail the run -- so a reply that is missing, unparseable or reports nothing means not recorded.+// Exported for tests.+export function hookRecorded(reply: string): boolean {+ try {+ const parsed: unknown = JSON.parse(reply)+ return typeof parsed === "object" && parsed !== null && "recorded" in parsed &&+ typeof parsed.recorded === "number" && parsed.recorded > 0+ } catch {+ return false+ }+}++// sendToBeacon resolves true only when the hook binary reports it recorded the event. Every failure+// is still swallowed -- telemetry never interrupts the run -- but a caller that sends something once+// rather than per event needs to know whether to try again.async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean> {// Flattened before anything else sees it, so every consumer of this function is handed plain// data rather than a live event with cycles and functions still attached.@@ -278,7 +307,7 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>let childtry {child = spawn(beaconArgv[0], beaconArgv.slice(1), {- stdio: ["pipe", "ignore", "ignore"],+ stdio: ["pipe", "pipe", "ignore"],windowsHide: true,})} catch (err) {@@ -305,7 +334,13 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>debugLog("hook binary failed", err)done(false)})- child.on("close", (code: number | null) => done(code === 0))+ let reply = ""+ child.stdout?.on("data", (chunk: Buffer) => {+ if (reply.length < maxReplyBytes) reply += chunk.toString("utf8")+ })+ child.stdout?.on("error", () => {})+ // `close` fires after the child's stdio has closed, so the whole reply has arrived.+ child.on("close", (code: number | null) => done(code === 0 && hookRecorded(reply)))// Without this, a hook binary that exits before reading stdin turns an EPIPE into an// unhandled error event in the host process. Beacon telemetry must never do that.child.stdin?.on("error", () => {})@@ -392,16 +427,25 @@ export function createBeaconExtension() {const fields = identity(ctx)if (event.type === "context") {// Only the listing leaves the extension, and only when it changed: `context` fires before- // every model call and carries the whole conversation. It is remembered after a send that- // landed, so a failed send is retried on the next call instead of losing the listing.+ // every model call and carries the whole conversation. A listing the hook did not record+ // is sent again on the next call, so one failed send does not lose it for the session.const listing = skillListing(ctx)// A session without an id is remembered under "", so it is still sent once rather than// before every model call.const key = typeof fields.sessionId === "string" ? fields.sessionId : ""- if (!listing || rememberedSkillListings.get(key) === listing) return- if (await sendToBeacon({ ...fields, type: "context", skillListing: listing })) {- rememberSkillListing(rememberedSkillListings, key, listing)+ const remembered = rememberedSkillListings.get(key)+ const same = remembered?.listing === listing+ if (!listing || (same && remembered?.done)) return+ const attempts = same && remembered ? remembered.attempts + 1 : 1+ const recorded = await sendToBeacon({ ...fields, type: "context", skillListing: listing })+ if (!recorded && attempts >= maxSkillListingAttempts) {+ debugLog("skill listing was not recorded; not sending it again", { attempts })}+ rememberSkillListing(rememberedSkillListings, key, {+ listing,+ attempts,+ done: recorded || attempts >= maxSkillListingAttempts,+ })return}// The event is spread last so a field it carries itself wins over the context's. `user_bash`diff --git a/plugins/omp-beacon/src/beacon.test.ts b/plugins/omp-beacon/src/beacon.test.tsindex 8821830d..dc26c873 100644--- a/plugins/omp-beacon/src/beacon.test.ts+++ b/plugins/omp-beacon/src/beacon.test.ts@@ -1,12 +1,12 @@import { afterEach, describe, expect, test } from "bun:test"-import { createBeaconExtension } from "./beacon"+import { createBeaconExtension, hookRecorded } from "./beacon"const senderKey = Symbol.for("beacon.omp.testSender")type Sent = Record<string, unknown>-// accepted decides what each send reports back, standing in for whether the hook binary took it.+// accepted decides what each send reports back, standing in for whether the hook binary recorded it.function captureSends(accepted: () => boolean = () => true): Sent[] {const sent: Sent[] = [];(globalThis as Record<symbol, unknown>)[senderKey] = (payload: Sent) => {@@ -203,6 +203,33 @@ describe("beacon oh my pi extension", () => {expect(sent[2].skillListing).toBe(listing)})+ // A hook that can never record the listing -- no endpoint log, an older hook binary -- must not+ // cost a spawn before every model call. A changed listing is a new listing and is tried again.+ test("stops resending a listing the hook never records, and tries a changed one", async () => {+ const sent = captureSends(() => false)+ const omp = fakeOmp()+ createBeaconExtension().register(omp.api)+ let listing = "- deploy: Deploy applications."+ const ctx = session("sess-never", { getSystemPrompt: () => [runtimeSkills(listing)] })++ for (let i = 0; i < 6; i++) await omp.fire({ type: "context", messages: [] }, ctx)+ expect(sent).toHaveLength(3)++ listing = "- review: Review a diff."+ await omp.fire({ type: "context", messages: [] }, ctx)+ expect(sent).toHaveLength(4)+ expect(sent[3].skillListing).toBe(listing)+ })++ // The hook exits 0 whether or not it wrote anything, so only its reply says the event landed.+ test("reads only a reply reporting a recorded event as delivered", () => {+ expect(hookRecorded('{"recorded":1}\n')).toBe(true)+ expect(hookRecorded('{"recorded":2}')).toBe(true)+ for (const reply of ['{"recorded":0}', "{}", "", "not json", '{"recorded":"1"}', "null", "1"]) {+ expect(hookRecorded(reply)).toBe(false)+ }+ })+test("sends a listing once for a session without an id", async () => {const sent = captureSends()const omp = fakeOmp()diff --git a/plugins/omp-beacon/src/beacon.ts b/plugins/omp-beacon/src/beacon.tsindex fbac406e..c02dec5e 100644--- a/plugins/omp-beacon/src/beacon.ts+++ b/plugins/omp-beacon/src/beacon.ts@@ -94,6 +94,18 @@ const skillsSection = /^# Skills & Rules\n(?:[^\n]*\n){0,3}?<skills>\s*([\s\S]*?// How many sessions' last-sent listings are remembered, least recently sent evicted first.const maxRememberedSkillSessions = 64+// A listing the hook did not record is sent again on later model calls, up to this many attempts+// in all. The cap keeps a hook that can never record it -- no endpoint log configured, a hook binary+// older than this extension -- from costing a spawn before every model call.+const maxSkillListingAttempts = 3++interface RememberedSkillListing {+ listing: string+ attempts: number+ // Recorded, or given up on after maxSkillListingAttempts.+ done: boolean+}+function skillListing(ctx: OmpContext | undefined): string {try {const prompt = ctx?.getSystemPrompt?.()@@ -110,7 +122,7 @@ function skillListing(ctx: OmpContext | undefined): string {return ""}-function rememberSkillListing(seen: Map<string, string>, key: string, listing: string) {+function rememberSkillListing(seen: Map<string, RememberedSkillListing>, key: string, listing: RememberedSkillListing) {// Re-inserted rather than updated so the map's insertion order is send order, and eviction// drops the session that has gone longest without a change.seen.delete(key)@@ -142,7 +154,7 @@ const maxPendingToolCalls = 64// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,// not a timestamp or a guess.const pendingToolCalls = new Map<string, unknown>()-const rememberedSkillListings = new Map<string, string>()+const rememberedSkillListings = new Map<string, RememberedSkillListing>()function rememberToolCall(event: OmpEvent) {const id = typeof event.toolCallId === "string" ? event.toolCallId : ""@@ -241,9 +253,26 @@ function safeClone(value: unknown, depth = 0, seen = new WeakSet<object>()): unk}}-// sendToBeacon resolves true only when the hook binary read the event and exited cleanly. Every-// failure is still swallowed -- telemetry never interrupts the run -- but a caller that sends-// something once rather than per event needs to know whether to try again.+// The most of the hook binary's reply that is read. The reply is one small object, {"recorded": n}.+const maxReplyBytes = 4096++// hookRecorded reads the hook binary's reply: whether it wrote at least one event. The exit status+// cannot say, because the hook exits 0 whether or not it wrote anything -- a hook that failed must+// not fail the run -- so a reply that is missing, unparseable or reports nothing means not recorded.+// Exported for tests.+export function hookRecorded(reply: string): boolean {+ try {+ const parsed: unknown = JSON.parse(reply)+ return typeof parsed === "object" && parsed !== null && "recorded" in parsed &&+ typeof parsed.recorded === "number" && parsed.recorded > 0+ } catch {+ return false+ }+}++// sendToBeacon resolves true only when the hook binary reports it recorded the event. Every failure+// is still swallowed -- telemetry never interrupts the run -- but a caller that sends something once+// rather than per event needs to know whether to try again.async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean> {// Flattened before anything else sees it, so every consumer of this function is handed plain// data rather than a live event with cycles and functions still attached.@@ -278,7 +307,7 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>let childtry {child = spawn(beaconArgv[0], beaconArgv.slice(1), {- stdio: ["pipe", "ignore", "ignore"],+ stdio: ["pipe", "pipe", "ignore"],windowsHide: true,})} catch (err) {@@ -305,7 +334,13 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>debugLog("hook binary failed", err)done(false)})- child.on("close", (code: number | null) => done(code === 0))+ let reply = ""+ child.stdout?.on("data", (chunk: Buffer) => {+ if (reply.length < maxReplyBytes) reply += chunk.toString("utf8")+ })+ child.stdout?.on("error", () => {})+ // `close` fires after the child's stdio has closed, so the whole reply has arrived.+ child.on("close", (code: number | null) => done(code === 0 && hookRecorded(reply)))// Without this, a hook binary that exits before reading stdin turns an EPIPE into an// unhandled error event in the host process. Beacon telemetry must never do that.child.stdin?.on("error", () => {})@@ -392,16 +427,25 @@ export function createBeaconExtension() {const fields = identity(ctx)if (event.type === "context") {// Only the listing leaves the extension, and only when it changed: `context` fires before- // every model call and carries the whole conversation. It is remembered after a send that- // landed, so a failed send is retried on the next call instead of losing the listing.+ // every model call and carries the whole conversation. A listing the hook did not record+ // is sent again on the next call, so one failed send does not lose it for the session.const listing = skillListing(ctx)// A session without an id is remembered under "", so it is still sent once rather than// before every model call.const key = typeof fields.sessionId === "string" ? fields.sessionId : ""- if (!listing || rememberedSkillListings.get(key) === listing) return- if (await sendToBeacon({ ...fields, type: "context", skillListing: listing })) {- rememberSkillListing(rememberedSkillListings, key, listing)+ const remembered = rememberedSkillListings.get(key)+ const same = remembered?.listing === listing+ if (!listing || (same && remembered?.done)) return+ const attempts = same && remembered ? remembered.attempts + 1 : 1+ const recorded = await sendToBeacon({ ...fields, type: "context", skillListing: listing })+ if (!recorded && attempts >= maxSkillListingAttempts) {+ debugLog("skill listing was not recorded; not sending it again", { attempts })}+ rememberSkillListing(rememberedSkillListings, key, {+ listing,+ attempts,+ done: recorded || attempts >= maxSkillListingAttempts,+ })return}// The event is spread last so a field it carries itself wins over the context's. `user_bash`