Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(omp): resend a skill listing until the hook reports it recorded

Kyle Elliott35dfb16c0f77
diff --git a/cli/beacon-hooks/cmd/omp_event_test.go b/cli/beacon-hooks/cmd/omp_event_test.go
index 1930a569..61a0af5b 100644
--- a/cli/beacon-hooks/cmd/omp_event_test.go
+++ b/cli/beacon-hooks/cmd/omp_event_test.go
@@ -1,6 +1,7 @@
package cmd
import (
+ "os"
"path/filepath"
"reflect"
"strings"
@@ -248,6 +249,45 @@ func TestOmpEventSkillListingIsWrittenAsSystemContext(t *testing.T) {
}
}
+// The extension decides whether to resend a skill listing from this reply, because the hook exits 0
+// whether or not it wrote anything. Every way an event is dropped must report nothing recorded.
+func TestOmpEventReplyReportsWhatWasRecorded(t *testing.T) {
+ listing := map[string]interface{}{"type": "context", "sessionId": "sess-1", "skillListing": "- deploy: Deploy applications."}
+ noLog := func(t *testing.T) {
+ for _, key := range []string{"BEACON_ENDPOINT_LOG", "BEACON_CLOUD_LOG_PATH", "BEACON_LOG_PATH", "BEACON_RUNTIME_LOG", "BEACON_ENDPOINT_MODE"} {
+ t.Setenv(key, "")
+ }
+ }
+ unwritable := func(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.Mkdir(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("BEACON_ENDPOINT_LOG", dir)
+ }
+ for _, tc := range []struct {
+ name string
+ input map[string]interface{}
+ setup func(*testing.T)
+ written float64
+ }{
+ {"written", listing, nil, 1},
+ {"log not writable", listing, unwritable, 0},
+ {"no endpoint log configured", listing, noLog, 0},
+ {"type this build does not map", map[string]interface{}{"type": "message_update", "sessionId": "sess-1"}, nil, 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ ompTestLog(t)
+ if tc.setup != nil {
+ tc.setup(t)
+ }
+ if out := runHookWithInput(t, runOmpEvent, tc.input); out["recorded"] != tc.written {
+ t.Fatalf("reply = %v, want recorded = %v", out, tc.written)
+ }
+ })
+ }
+}
+
// Print and ACP never emit `input`; the extension forwards the delivered user message instead,
// with its text under `prompt`. The row names that origin, because the text is what Oh My Pi sent
// the model after expanding templates and commands, not what an `input` event would carry.
diff --git a/cli/beacon-hooks/cmd/pi_event.go b/cli/beacon-hooks/cmd/pi_event.go
index bffac5d5..cc4de2c5 100644
--- a/cli/beacon-hooks/cmd/pi_event.go
+++ b/cli/beacon-hooks/cmd/pi_event.go
@@ -2,6 +2,8 @@ package cmd
import (
"github.com/spf13/cobra"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"
)
// pi-event is the single entry point for every Pi lifecycle payload.
@@ -32,21 +34,30 @@ func runPiEvent(cmd *cobra.Command, args []string) {
//
// Shared by pi-event and omp-event because the transport is identical -- the extension spawns the
// hook binary and writes one JSON object to its stdin -- and only the runtime it describes differs.
+//
+// The reply, {"recorded": n}, is how many events reached the log, a duplicate the log already holds
+// included. It is the only way the extension can learn that: the exit status is 0 whether or not
+// anything was written, because a hook that fails must not fail the run it observes, and an
+// envelope that did not parse, a log that cannot be written, a log that is not configured and an
+// event type this build does not map all end the same way.
func runPiFamilyEvent(runtime piFamily) {
input, err := readStdinJSON()
if err != nil {
- outputJSON(emptyResponse)
+ outputJSON(map[string]interface{}{"recorded": 0})
return
}
sessionID := resolveSessionID(input, runtime.platform)
logger := newHookLogger(runtime.platform+"-event", runtime.platform, sessionID)
+ recorded := 0
for _, event := range runtime.endpointEvents(input, sessionID) {
if event.action == "" {
continue
}
- _ = logger.EndpointEvent(event.action, event.category, event.severity, event.message, event.fields)
+ if logger.EndpointEvent(event.action, event.category, event.severity, event.message, event.fields) == nil && logging.EndpointLogConfigured() {
+ recorded++
+ }
}
- outputJSON(emptyResponse)
+ outputJSON(map[string]interface{}{"recorded": recorded})
}
// supportedPiEventTypes lists every Pi event type this mapper handles.
diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.go
index 1bef5977..9d2e70e5 100644
--- a/cli/beacon-hooks/internal/logging/logging.go
+++ b/cli/beacon-hooks/internal/logging/logging.go
@@ -514,6 +514,13 @@ func minimalEndpointEvent(event map[string]interface{}) map[string]interface{} {
return out
}
+// EndpointLogConfigured reports whether this process has a runtime log to write to. Without one,
+// EndpointEvent records nothing and returns nil, so a caller reporting what it recorded checks this
+// rather than reading a nil error as a write.
+func EndpointLogConfigured() bool {
+ return endpointLogPath() != ""
+}
+
func endpointLogPath() string {
if path := firstEnv("BEACON_ENDPOINT_LOG", "BEACON_CLOUD_LOG_PATH", "BEACON_LOG_PATH", "BEACON_RUNTIME_LOG"); path != "" {
return path
diff --git a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
index fbac406e..c02dec5e 100644
--- a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
+++ b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
@@ -94,6 +94,18 @@ const skillsSection = /^# Skills & Rules\n(?:[^\n]*\n){0,3}?<skills>\s*([\s\S]*?
// How many sessions' last-sent listings are remembered, least recently sent evicted first.
const maxRememberedSkillSessions = 64
+// A listing the hook did not record is sent again on later model calls, up to this many attempts
+// in all. The cap keeps a hook that can never record it -- no endpoint log configured, a hook binary
+// older than this extension -- from costing a spawn before every model call.
+const maxSkillListingAttempts = 3
+
+interface RememberedSkillListing {
+ listing: string
+ attempts: number
+ // Recorded, or given up on after maxSkillListingAttempts.
+ done: boolean
+}
+
function skillListing(ctx: OmpContext | undefined): string {
try {
const prompt = ctx?.getSystemPrompt?.()
@@ -110,7 +122,7 @@ function skillListing(ctx: OmpContext | undefined): string {
return ""
}
-function rememberSkillListing(seen: Map<string, string>, key: string, listing: string) {
+function rememberSkillListing(seen: Map<string, RememberedSkillListing>, key: string, listing: RememberedSkillListing) {
// Re-inserted rather than updated so the map's insertion order is send order, and eviction
// drops the session that has gone longest without a change.
seen.delete(key)
@@ -142,7 +154,7 @@ const maxPendingToolCalls = 64
// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,
// not a timestamp or a guess.
const pendingToolCalls = new Map<string, unknown>()
-const rememberedSkillListings = new Map<string, string>()
+const rememberedSkillListings = new Map<string, RememberedSkillListing>()
function rememberToolCall(event: OmpEvent) {
const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
@@ -241,9 +253,26 @@ function safeClone(value: unknown, depth = 0, seen = new WeakSet<object>()): unk
}
}
-// sendToBeacon resolves true only when the hook binary read the event and exited cleanly. Every
-// failure is still swallowed -- telemetry never interrupts the run -- but a caller that sends
-// something once rather than per event needs to know whether to try again.
+// The most of the hook binary's reply that is read. The reply is one small object, {"recorded": n}.
+const maxReplyBytes = 4096
+
+// hookRecorded reads the hook binary's reply: whether it wrote at least one event. The exit status
+// cannot say, because the hook exits 0 whether or not it wrote anything -- a hook that failed must
+// not fail the run -- so a reply that is missing, unparseable or reports nothing means not recorded.
+// Exported for tests.
+export function hookRecorded(reply: string): boolean {
+ try {
+ const parsed: unknown = JSON.parse(reply)
+ return typeof parsed === "object" && parsed !== null && "recorded" in parsed &&
+ typeof parsed.recorded === "number" && parsed.recorded > 0
+ } catch {
+ return false
+ }
+}
+
+// sendToBeacon resolves true only when the hook binary reports it recorded the event. Every failure
+// is still swallowed -- telemetry never interrupts the run -- but a caller that sends something once
+// rather than per event needs to know whether to try again.
async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean> {
// Flattened before anything else sees it, so every consumer of this function is handed plain
// data rather than a live event with cycles and functions still attached.
@@ -278,7 +307,7 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>
let child
try {
child = spawn(beaconArgv[0], beaconArgv.slice(1), {
- stdio: ["pipe", "ignore", "ignore"],
+ stdio: ["pipe", "pipe", "ignore"],
windowsHide: true,
})
} catch (err) {
@@ -305,7 +334,13 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>
debugLog("hook binary failed", err)
done(false)
})
- child.on("close", (code: number | null) => done(code === 0))
+ let reply = ""
+ child.stdout?.on("data", (chunk: Buffer) => {
+ if (reply.length < maxReplyBytes) reply += chunk.toString("utf8")
+ })
+ child.stdout?.on("error", () => {})
+ // `close` fires after the child's stdio has closed, so the whole reply has arrived.
+ child.on("close", (code: number | null) => done(code === 0 && hookRecorded(reply)))
// Without this, a hook binary that exits before reading stdin turns an EPIPE into an
// unhandled error event in the host process. Beacon telemetry must never do that.
child.stdin?.on("error", () => {})
@@ -392,16 +427,25 @@ export function createBeaconExtension() {
const fields = identity(ctx)
if (event.type === "context") {
// Only the listing leaves the extension, and only when it changed: `context` fires before
- // every model call and carries the whole conversation. It is remembered after a send that
- // landed, so a failed send is retried on the next call instead of losing the listing.
+ // every model call and carries the whole conversation. A listing the hook did not record
+ // is sent again on the next call, so one failed send does not lose it for the session.
const listing = skillListing(ctx)
// A session without an id is remembered under "", so it is still sent once rather than
// before every model call.
const key = typeof fields.sessionId === "string" ? fields.sessionId : ""
- if (!listing || rememberedSkillListings.get(key) === listing) return
- if (await sendToBeacon({ ...fields, type: "context", skillListing: listing })) {
- rememberSkillListing(rememberedSkillListings, key, listing)
+ const remembered = rememberedSkillListings.get(key)
+ const same = remembered?.listing === listing
+ if (!listing || (same && remembered?.done)) return
+ const attempts = same && remembered ? remembered.attempts + 1 : 1
+ const recorded = await sendToBeacon({ ...fields, type: "context", skillListing: listing })
+ if (!recorded && attempts >= maxSkillListingAttempts) {
+ debugLog("skill listing was not recorded; not sending it again", { attempts })
}
+ rememberSkillListing(rememberedSkillListings, key, {
+ listing,
+ attempts,
+ done: recorded || attempts >= maxSkillListingAttempts,
+ })
return
}
// The event is spread last so a field it carries itself wins over the context's. `user_bash`
diff --git a/plugins/omp-beacon/src/beacon.test.ts b/plugins/omp-beacon/src/beacon.test.ts
index 8821830d..dc26c873 100644
--- a/plugins/omp-beacon/src/beacon.test.ts
+++ b/plugins/omp-beacon/src/beacon.test.ts
@@ -1,12 +1,12 @@
import { afterEach, describe, expect, test } from "bun:test"
-import { createBeaconExtension } from "./beacon"
+import { createBeaconExtension, hookRecorded } from "./beacon"
const senderKey = Symbol.for("beacon.omp.testSender")
type Sent = Record<string, unknown>
-// accepted decides what each send reports back, standing in for whether the hook binary took it.
+// accepted decides what each send reports back, standing in for whether the hook binary recorded it.
function captureSends(accepted: () => boolean = () => true): Sent[] {
const sent: Sent[] = []
;(globalThis as Record<symbol, unknown>)[senderKey] = (payload: Sent) => {
@@ -203,6 +203,33 @@ describe("beacon oh my pi extension", () => {
expect(sent[2].skillListing).toBe(listing)
})
+ // A hook that can never record the listing -- no endpoint log, an older hook binary -- must not
+ // cost a spawn before every model call. A changed listing is a new listing and is tried again.
+ test("stops resending a listing the hook never records, and tries a changed one", async () => {
+ const sent = captureSends(() => false)
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+ let listing = "- deploy: Deploy applications."
+ const ctx = session("sess-never", { getSystemPrompt: () => [runtimeSkills(listing)] })
+
+ for (let i = 0; i < 6; i++) await omp.fire({ type: "context", messages: [] }, ctx)
+ expect(sent).toHaveLength(3)
+
+ listing = "- review: Review a diff."
+ await omp.fire({ type: "context", messages: [] }, ctx)
+ expect(sent).toHaveLength(4)
+ expect(sent[3].skillListing).toBe(listing)
+ })
+
+ // The hook exits 0 whether or not it wrote anything, so only its reply says the event landed.
+ test("reads only a reply reporting a recorded event as delivered", () => {
+ expect(hookRecorded('{"recorded":1}\n')).toBe(true)
+ expect(hookRecorded('{"recorded":2}')).toBe(true)
+ for (const reply of ['{"recorded":0}', "{}", "", "not json", '{"recorded":"1"}', "null", "1"]) {
+ expect(hookRecorded(reply)).toBe(false)
+ }
+ })
+
test("sends a listing once for a session without an id", async () => {
const sent = captureSends()
const omp = fakeOmp()
diff --git a/plugins/omp-beacon/src/beacon.ts b/plugins/omp-beacon/src/beacon.ts
index fbac406e..c02dec5e 100644
--- a/plugins/omp-beacon/src/beacon.ts
+++ b/plugins/omp-beacon/src/beacon.ts
@@ -94,6 +94,18 @@ const skillsSection = /^# Skills & Rules\n(?:[^\n]*\n){0,3}?<skills>\s*([\s\S]*?
// How many sessions' last-sent listings are remembered, least recently sent evicted first.
const maxRememberedSkillSessions = 64
+// A listing the hook did not record is sent again on later model calls, up to this many attempts
+// in all. The cap keeps a hook that can never record it -- no endpoint log configured, a hook binary
+// older than this extension -- from costing a spawn before every model call.
+const maxSkillListingAttempts = 3
+
+interface RememberedSkillListing {
+ listing: string
+ attempts: number
+ // Recorded, or given up on after maxSkillListingAttempts.
+ done: boolean
+}
+
function skillListing(ctx: OmpContext | undefined): string {
try {
const prompt = ctx?.getSystemPrompt?.()
@@ -110,7 +122,7 @@ function skillListing(ctx: OmpContext | undefined): string {
return ""
}
-function rememberSkillListing(seen: Map<string, string>, key: string, listing: string) {
+function rememberSkillListing(seen: Map<string, RememberedSkillListing>, key: string, listing: RememberedSkillListing) {
// Re-inserted rather than updated so the map's insertion order is send order, and eviction
// drops the session that has gone longest without a change.
seen.delete(key)
@@ -142,7 +154,7 @@ const maxPendingToolCalls = 64
// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,
// not a timestamp or a guess.
const pendingToolCalls = new Map<string, unknown>()
-const rememberedSkillListings = new Map<string, string>()
+const rememberedSkillListings = new Map<string, RememberedSkillListing>()
function rememberToolCall(event: OmpEvent) {
const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
@@ -241,9 +253,26 @@ function safeClone(value: unknown, depth = 0, seen = new WeakSet<object>()): unk
}
}
-// sendToBeacon resolves true only when the hook binary read the event and exited cleanly. Every
-// failure is still swallowed -- telemetry never interrupts the run -- but a caller that sends
-// something once rather than per event needs to know whether to try again.
+// The most of the hook binary's reply that is read. The reply is one small object, {"recorded": n}.
+const maxReplyBytes = 4096
+
+// hookRecorded reads the hook binary's reply: whether it wrote at least one event. The exit status
+// cannot say, because the hook exits 0 whether or not it wrote anything -- a hook that failed must
+// not fail the run -- so a reply that is missing, unparseable or reports nothing means not recorded.
+// Exported for tests.
+export function hookRecorded(reply: string): boolean {
+ try {
+ const parsed: unknown = JSON.parse(reply)
+ return typeof parsed === "object" && parsed !== null && "recorded" in parsed &&
+ typeof parsed.recorded === "number" && parsed.recorded > 0
+ } catch {
+ return false
+ }
+}
+
+// sendToBeacon resolves true only when the hook binary reports it recorded the event. Every failure
+// is still swallowed -- telemetry never interrupts the run -- but a caller that sends something once
+// rather than per event needs to know whether to try again.
async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean> {
// Flattened before anything else sees it, so every consumer of this function is handed plain
// data rather than a live event with cycles and functions still attached.
@@ -278,7 +307,7 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>
let child
try {
child = spawn(beaconArgv[0], beaconArgv.slice(1), {
- stdio: ["pipe", "ignore", "ignore"],
+ stdio: ["pipe", "pipe", "ignore"],
windowsHide: true,
})
} catch (err) {
@@ -305,7 +334,13 @@ async function sendToBeacon(payload: Record<string, unknown>): Promise<boolean>
debugLog("hook binary failed", err)
done(false)
})
- child.on("close", (code: number | null) => done(code === 0))
+ let reply = ""
+ child.stdout?.on("data", (chunk: Buffer) => {
+ if (reply.length < maxReplyBytes) reply += chunk.toString("utf8")
+ })
+ child.stdout?.on("error", () => {})
+ // `close` fires after the child's stdio has closed, so the whole reply has arrived.
+ child.on("close", (code: number | null) => done(code === 0 && hookRecorded(reply)))
// Without this, a hook binary that exits before reading stdin turns an EPIPE into an
// unhandled error event in the host process. Beacon telemetry must never do that.
child.stdin?.on("error", () => {})
@@ -392,16 +427,25 @@ export function createBeaconExtension() {
const fields = identity(ctx)
if (event.type === "context") {
// Only the listing leaves the extension, and only when it changed: `context` fires before
- // every model call and carries the whole conversation. It is remembered after a send that
- // landed, so a failed send is retried on the next call instead of losing the listing.
+ // every model call and carries the whole conversation. A listing the hook did not record
+ // is sent again on the next call, so one failed send does not lose it for the session.
const listing = skillListing(ctx)
// A session without an id is remembered under "", so it is still sent once rather than
// before every model call.
const key = typeof fields.sessionId === "string" ? fields.sessionId : ""
- if (!listing || rememberedSkillListings.get(key) === listing) return
- if (await sendToBeacon({ ...fields, type: "context", skillListing: listing })) {
- rememberSkillListing(rememberedSkillListings, key, listing)
+ const remembered = rememberedSkillListings.get(key)
+ const same = remembered?.listing === listing
+ if (!listing || (same && remembered?.done)) return
+ const attempts = same && remembered ? remembered.attempts + 1 : 1
+ const recorded = await sendToBeacon({ ...fields, type: "context", skillListing: listing })
+ if (!recorded && attempts >= maxSkillListingAttempts) {
+ debugLog("skill listing was not recorded; not sending it again", { attempts })
}
+ rememberSkillListing(rememberedSkillListings, key, {
+ listing,
+ attempts,
+ done: recorded || attempts >= maxSkillListingAttempts,
+ })
return
}
// The event is spread last so a field it carries itself wins over the context's. `user_bash`