Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Keep judgment catalog and verdicts consistent

Rach Pradhan5d6293f0d27f
diff --git a/src/agent_request.zig b/src/agent_request.zig
index b648f41c..a4ae4e05 100644
--- a/src/agent_request.zig
+++ b/src/agent_request.zig
@@ -119,7 +119,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
// so the model lands a text answer now instead of asking for a tool the
// budget can never pay for — which is how the audit smoke died narrating.
// compaction/title requests pass tools=null already and skip this whole.
- var tools = tools_in;
+ var tools = try @import("jev_tool.zig").refreshCatalogForRequest(self, tools_in);
if (self.tracer) |tr| {
if (tools) |t| if (t.len == 0) tr.note("tools", "empty catalog at request time (#695)");
}
diff --git a/src/exec.zig b/src/exec.zig
index ff51bc2b..da76af85 100644
--- a/src/exec.zig
+++ b/src/exec.zig
@@ -443,6 +443,11 @@ test "native Jev dispatch rejects other models and latches off after one failed
try std.testing.expect(older_model.is_error);
ctx.provider.model = "gpt-6-sol";
try std.testing.expect(jev.available(ctx.provider)); // ineligible attempts did not trip the circuit
+ ctx.from_sub = true;
+ const worker = try execToolInner(ctx, call);
+ defer std.testing.allocator.free(worker.text);
+ try std.testing.expect(worker.is_error);
+ ctx.from_sub = false;
const first = try execToolInner(ctx, call);
defer std.testing.allocator.free(first.text);
const second = try execToolInner(ctx, call);
diff --git a/src/jev_provider_switch_tests.zig b/src/jev_provider_switch_tests.zig
index 3f4cf4b0..a7290791 100644
--- a/src/jev_provider_switch_tests.zig
+++ b/src/jev_provider_switch_tests.zig
@@ -68,4 +68,25 @@ test "native Jev catalog refreshes on eligible and ineligible model switches" {
try std.testing.expect(std.mem.indexOf(u8, root.tools_openai, jev_tool.name) == null);
_ = try providers.applyProviderInner(&root, a, gpt6, false);
try std.testing.expect(std.mem.indexOf(u8, root.tools_responses, jev_tool.name) != null);
+ // The OpenAI-format slot was cached above without Jev. Switching from
+ // one eligible provider to another must rebuild that older slot.
+ const eligible_openai = mimo;
+ _ = try providers.applyProviderInner(&root, a, eligible_openai, false);
+ try std.testing.expect(std.mem.indexOf(u8, root.tools_openai, jev_tool.name) != null);
+
+ jev_tool.configure(struct {
+ pub fn get(_: @This(), key: []const u8) ?[]const u8 {
+ return if (std.mem.eql(u8, key, "JEV_BACKEND")) "mock-fail" else null;
+ }
+ }{});
+ _ = jev_tool.setCodegraffLogin(true);
+ const old_catalog = root.toolsJson();
+ const input = try std.json.parseFromSliceLeaky(std.json.Value, a, "{\"state\":\"10 tests passed\",\"question\":\"Did CI pass?\",\"type\":\"noul\"}", .{});
+ var client: std.http.Client = undefined;
+ const ctx: @import("tools.zig").ToolCtx = .{ .gpa = std.testing.allocator, .io = std.testing.io, .client = &client, .provider = eligible_openai, .registry = null, .from_sub = false, .approvals = null, .tracer = null };
+ const failed = try jev_tool.execute(ctx, input);
+ defer std.testing.allocator.free(failed.text);
+ const next_catalog = (try jev_tool.refreshCatalogForRequest(&root, old_catalog)).?;
+ try std.testing.expect(std.mem.indexOf(u8, next_catalog, jev_tool.name) == null);
+ try std.testing.expect(std.mem.indexOf(u8, old_catalog, jev_tool.name) != null);
}
diff --git a/src/jev_tool.zig b/src/jev_tool.zig
index dfa5d89c..b2ee6cf7 100644
--- a/src/jev_tool.zig
+++ b/src/jev_tool.zig
@@ -14,7 +14,7 @@ const scope = @import("jev_model_scope.zig");
pub const name = "jev_judge";
pub const description = "Ask Jev one closed-form judgment about a SHORT, NON-SENSITIVE state. Requires a Codegraff login and an eligible GPT-6 or MiMo model. Use for yes/no (noul), a choice, or an ordered score, not writing or open-ended reasoning. Never send source code, secrets, customer data, paths, or unrelated context. A low-confidence verdict escalates to you. If Jev fails once, this tool skips all later Jev calls for this session; decide yourself instead.";
pub const input_schema =
- \\{"type":"object","properties":{"state":{"type":"string","description":"Short non-sensitive facts needed for this judgment only; no code, paths or secrets"},"question":{"type":"string","description":"One closed-form question about state"},"type":{"type":"string","enum":["noul","choice","score"],"description":"noul=yes/no probability; choice=one option; score=ordered level"},"options":{"type":"array","items":{"type":"string"},"description":"Required for choice: 2-16 distinct labels"},"levels":{"type":"array","items":{"type":"string"},"description":"Required for score: 2-16 ordered descriptions"}},"required":["state","question","type"]}
+ \\{"type":"object","properties":{"state":{"type":"string","description":"Short non-sensitive facts needed for this judgment only; no code, paths or secrets"},"question":{"type":"string","description":"One closed-form question about state"},"type":{"type":"string","enum":["noul","choice","score"],"description":"noul=yes/no probability; choice=one option; score=ordered level"},"options":{"type":"array","items":{"type":"string"},"description":"Required for choice: 2-16 distinct labels"},"levels":{"type":"array","items":{"type":"string"},"description":"Required for score: 2-10 ordered descriptions"}},"required":["state","question","type"]}
;
const spec = ToolSpec{ .name = name, .desc = description, .schema = input_schema };
const endpoint = "https://api.typesafe.ai/v1/systemone";
@@ -56,7 +56,11 @@ pub fn available(provider: Provider) bool {
/// A model switch can change Jev visibility even when the wire format stays the same.
pub fn updateProvider(root: anytype, p: Provider) void {
- if (available(root.provider) != available(p)) root.invalidateRootTools();
+ // Each wire format has its own cached catalog. An older catalog for the
+ // destination format may have been built before Jev became available.
+ if (available(root.provider) != available(p) or
+ (root.provider.kind != p.kind and (available(root.provider) or available(p))))
+ root.invalidateRootTools();
root.provider = p;
}
@@ -68,6 +72,15 @@ pub fn takeCatalogRefresh() bool {
return state.refresh.swap(false, .acq_rel);
}
+pub fn refreshCatalogForRequest(root: anytype, tools_in: ?[]const u8) !?[]const u8 {
+ // RLM calls host tools directly, so runTools may not observe a failed
+ // Jev call. Replace the caller's pre-refresh catalog snapshot here.
+ if (root.sub or !takeCatalogRefresh()) return tools_in;
+ root.invalidateRootTools();
+ try root.ensureRootTools(root.provider.kind);
+ return if (tools_in == null) null else root.toolsJson();
+}
+
fn skipped(gpa: Allocator) !ToolOutput {
return .{ .text = try gpa.dupe(u8, skip_text) };
}
@@ -88,8 +101,8 @@ fn number(v: Value) ?f64 {
};
}
-fn listValid(v: Value) bool {
- if (v != .array or v.array.items.len < 2 or v.array.items.len > 16) return false;
+fn listValid(v: Value, max: usize) bool {
+ if (v != .array or v.array.items.len < 2 or v.array.items.len > max) return false;
for (v.array.items, 0..) |item, i| {
const label = string(item) orelse return false;
if (label.len == 0 or label.len > 120) return false;
@@ -110,13 +123,13 @@ fn makeBody(arena: Allocator, input: Value) ![]const u8 {
try q.put(arena, "instructions", .{ .string = question });
if (std.mem.eql(u8, kind, "choice")) {
const options = obj.get("options") orelse return error.InvalidInput;
- if (!listValid(options)) return error.InvalidInput;
+ if (!listValid(options, 16)) return error.InvalidInput;
var labels = std.json.ObjectMap.empty;
for (options.array.items) |item| try labels.put(arena, item.string, item);
try q.put(arena, "criteria", .{ .object = labels });
} else if (std.mem.eql(u8, kind, "score")) {
const levels = obj.get("levels") orelse return error.InvalidInput;
- if (!listValid(levels)) return error.InvalidInput;
+ if (!listValid(levels, 10)) return error.InvalidInput;
try q.put(arena, "criteria", levels);
} else if (!std.mem.eql(u8, kind, "noul")) return error.InvalidInput;
var questions = std.json.ObjectMap.empty;
@@ -133,31 +146,31 @@ fn makeBody(arena: Allocator, input: Value) ![]const u8 {
fn mockResponse(arena: Allocator, input: Value) ![]const u8 {
const kind = input.object.get("type").?.string;
- if (std.mem.eql(u8, kind, "noul")) return "{\"answers\":{\"q1\":{\"noul\":0.98}}}";
- if (std.mem.eql(u8, kind, "score")) return "{\"answers\":{\"q1\":{\"score\":0.9,\"confidence\":0.95}}}";
+ if (std.mem.eql(u8, kind, "noul")) return "{\"answers\":{\"q1\":{\"type\":\"noul\",\"noul\":0.98}}}";
+ if (std.mem.eql(u8, kind, "score")) return "{\"answers\":{\"q1\":{\"type\":\"score\",\"score\":0.9,\"confidence\":0.95}}}";
const label = input.object.get("options").?.array.items[0].string;
var aw: Io.Writer.Allocating = .init(arena);
var s: std.json.Stringify = .{ .writer = &aw.writer };
- try s.write(.{ .answers = .{ .q1 = .{ .choice = label, .confidence = 0.95 } } });
+ try s.write(.{ .answers = .{ .q1 = .{ .type = "choice", .choice = label, .confidence = 0.95 } } });
return aw.writer.buffered();
}
fn fetch(ctx: ToolCtx, arena: Allocator, body: []const u8) ![]const u8 {
const bearer = try std.fmt.allocPrint(arena, "Bearer {s}", .{state.key});
- var aw: Io.Writer.Allocating = .init(arena);
+ var aw = Io.Writer.fixed(try arena.alloc(u8, 64 * 1024));
const res = try ctx.client.fetch(.{
.location = .{ .url = endpoint },
.method = .POST,
.payload = body,
- .response_writer = &aw.writer,
+ .response_writer = &aw,
.redirect_behavior = .unhandled,
.headers = .{
.content_type = .{ .override = "application/json" },
.authorization = .{ .override = bearer },
},
});
- if (@intFromEnum(res.status) != 200 or aw.writer.buffered().len > 64 * 1024) return error.JevUnavailable;
- return aw.writer.buffered();
+ if (@intFromEnum(res.status) != 200) return error.JevUnavailable;
+ return aw.buffered();
}
fn verdict(arena: Allocator, input: Value, raw: []const u8) ![]const u8 {
@@ -168,6 +181,8 @@ fn verdict(arena: Allocator, input: Value, raw: []const u8) ![]const u8 {
const answer = answers.object.get("q1") orelse return error.InvalidResponse;
if (answer != .object) return error.InvalidResponse;
const kind = input.object.get("type").?.string;
+ const answer_type = string(answer.object.get("type") orelse return error.InvalidResponse) orelse return error.InvalidResponse;
+ if (!std.mem.eql(u8, kind, answer_type)) return error.InvalidResponse;
const value: Value = if (std.mem.eql(u8, kind, "noul")) answer.object.get("noul") orelse return error.InvalidResponse else if (std.mem.eql(u8, kind, "choice")) answer.object.get("choice") orelse return error.InvalidResponse else answer.object.get("score") orelse return error.InvalidResponse;
if (std.mem.eql(u8, kind, "choice")) {
const label = string(value) orelse return error.InvalidResponse;
@@ -180,9 +195,11 @@ fn verdict(arena: Allocator, input: Value, raw: []const u8) ![]const u8 {
if (!found) return error.InvalidResponse;
} else {
const n = number(value) orelse return error.InvalidResponse;
- if (n < 0 or (std.mem.eql(u8, kind, "noul") and n > 1)) return error.InvalidResponse;
+ const max: f64 = if (std.mem.eql(u8, kind, "score")) @floatFromInt(input.object.get("levels").?.array.items.len - 1) else 1;
+ if (n < 0 or n > max) return error.InvalidResponse;
}
- const reported = if (answer.object.get("confidence")) |v| number(v) else null;
+ const reported = if (answer.object.get("confidence")) |v| number(v) orelse return error.InvalidResponse else null;
+ if (!std.mem.eql(u8, kind, "noul") and reported == null) return error.InvalidResponse;
if (reported) |c| if (c < 0 or c > 1) return error.InvalidResponse;
const estimated: f64 = if (std.mem.eql(u8, kind, "noul")) 2 * @abs(number(value).? - 0.5) else 0;
const confidence = reported orelse estimated;
@@ -194,6 +211,7 @@ fn verdict(arena: Allocator, input: Value, raw: []const u8) ![]const u8 {
}
pub fn execute(ctx: ToolCtx, input: Value) !ToolOutput {
+ if (ctx.from_sub) return invalid(ctx.gpa, "jev_judge is available only to the root agent");
if (!state.codegraff_login.load(.acquire)) return invalid(ctx.gpa, "jev_judge requires a Codegraff login (`graff login`)");
if (!scope.eligible(ctx.provider)) return invalid(ctx.gpa, "jev_judge is available only with Codex/OpenAI GPT-6 or Xiaomi MiMo models");
if (state.backend == .typesafe and state.key.len == 0) return skipped(ctx.gpa);
@@ -292,3 +310,15 @@ test "native Jev mock uses TypeSafe wire and returns a typed verdict" {
try std.testing.expect(!result.object.get("escalate").?.bool);
try std.testing.expectEqual(@as(usize, 1), state.attempts.load(.acquire));
}
+
+test "native Jev rejects out-of-range scores and malformed confidence" {
+ var temp = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer temp.deinit();
+ const a = temp.allocator();
+ const input = try std.json.parseFromSliceLeaky(Value, a, "{\"state\":\"build passed\",\"question\":\"How complete?\",\"type\":\"score\",\"levels\":[\"none\",\"some\",\"all\"]}", .{});
+ try std.testing.expectError(error.InvalidResponse, verdict(a, input, "{\"answers\":{\"q1\":{\"type\":\"score\",\"score\":2.5,\"confidence\":0.9}}}"));
+ try std.testing.expectError(error.InvalidResponse, verdict(a, input, "{\"answers\":{\"q1\":{\"type\":\"score\",\"score\":1.5,\"confidence\":\"certain\"}}}"));
+ try std.testing.expectError(error.InvalidResponse, verdict(a, input, "{\"answers\":{\"q1\":{\"type\":\"choice\",\"score\":1.5,\"confidence\":0.9}}}"));
+ const good = try verdict(a, input, "{\"answers\":{\"q1\":{\"type\":\"score\",\"score\":1.5,\"confidence\":0.9}}}");
+ try std.testing.expect(std.mem.indexOf(u8, good, "\"answer\":1.5") != null);
+}