Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
feat(cast): add JSON control character sanitization for tool call arguments
Dmitry Ngf57e988586a5
diff --git a/backend/pkg/cast/chain_ast.go b/backend/pkg/cast/chain_ast.goindex 7445de29..cddbccc9 100644--- a/backend/pkg/cast/chain_ast.go+++ b/backend/pkg/cast/chain_ast.go@@ -1,6 +1,7 @@package castimport (+ "encoding/json""fmt""pentagi/pkg/templates""sort"@@ -1056,6 +1057,104 @@ func ContainsToolCallReasoning(messages []llms.MessageContent) bool {return false}+// SanitizeJSONControlChars escapes any literal control characters (0x00–0x1F, 0x7F) that+// appear inside JSON string values. Such characters are disallowed by the JSON spec and+// must be encoded as \n, \r, \t, or \uXXXX escape sequences.+//+// The scanner is context-aware: it tracks whether it is currently inside a JSON string and+// skips over already-escaped sequences so they are never double-escaped.+func SanitizeJSONControlChars(s string) string {+ if json.Valid([]byte(s)) {+ return s+ }++ var buf strings.Builder+ buf.Grow(len(s) + 32)++ inString := false+ escaped := false++ for i := 0; i < len(s); {+ b := s[i]++ if escaped {+ buf.WriteByte(b)+ i+++ escaped = false+ continue+ }++ if inString && b == '\\' {+ buf.WriteByte(b)+ i+++ escaped = true+ continue+ }++ if b == '"' {+ buf.WriteByte(b)+ i+++ inString = !inString+ continue+ }++ if inString && b < 0x20 {+ switch b {+ case '\n':+ buf.WriteString(`\n`)+ case '\r':+ buf.WriteString(`\r`)+ case '\t':+ buf.WriteString(`\t`)+ case '\b':+ buf.WriteString(`\b`)+ case '\f':+ buf.WriteString(`\f`)+ default:+ fmt.Fprintf(&buf, `\u%04x`, b)+ }+ i+++ continue+ }++ buf.WriteByte(b)+ i+++ }++ return buf.String()+}++// SanitizeToolCallArguments scans every tool call in the chain and ensures its Arguments+// field is valid JSON. Literal control characters that some LLM providers occasionally+// emit inside string values are escaped so that downstream consumers (e.g. vLLM's+// _postprocess_messages) can parse the arguments without errors.+func (ast *ChainAST) SanitizeToolCallArguments() {+ for _, section := range ast.Sections {+ for _, bodyPair := range section.Body {+ if bodyPair.Type != RequestResponse && bodyPair.Type != Summarization {+ continue+ }++ if bodyPair.AIMessage == nil {+ continue+ }++ for pdx, part := range bodyPair.AIMessage.Parts {+ toolCall, ok := part.(llms.ToolCall)+ if !ok || toolCall.FunctionCall == nil {+ continue+ }++ sanitized := SanitizeJSONControlChars(toolCall.FunctionCall.Arguments)+ if sanitized != toolCall.FunctionCall.Arguments {+ toolCall.FunctionCall.Arguments = sanitized+ bodyPair.AIMessage.Parts[pdx] = toolCall+ }+ }+ }+ }+}+// ExtractReasoningMessage extracts the first AI message that contains reasoning content// in a TextContent part. This is useful for preserving reasoning messages when summarizing// content for providers like Kimi (Moonshot) that require reasoning_content before ToolCall.diff --git a/backend/pkg/cast/chain_ast_test.go b/backend/pkg/cast/chain_ast_test.goindex 28f4ee51..edfe4fb0 100644--- a/backend/pkg/cast/chain_ast_test.go+++ b/backend/pkg/cast/chain_ast_test.go@@ -3015,3 +3015,178 @@ func TestClearReasoning_IntegrationWithNormalize(t *testing.T) {t.Log("Successfully normalized IDs and cleared reasoning for provider switch")}++func TestSanitizeJSONControlChars(t *testing.T) {+ tests := []struct {+ name string+ input string+ want string+ }{+ {+ name: "already valid JSON - no changes",+ input: `{"input": "hello world"}`,+ want: `{"input": "hello world"}`,+ },+ {+ name: "already valid JSON with escaped newline - no changes",+ input: `{"input": "line1\nline2"}`,+ want: `{"input": "line1\nline2"}`,+ },+ {+ name: "literal newline inside string value",+ input: "{\"input\": \"line1\nline2\"}",+ want: `{"input": "line1\nline2"}`,+ },+ {+ name: "literal carriage return inside string value",+ input: "{\"cmd\": \"echo\rtest\"}",+ want: `{"cmd": "echo\rtest"}`,+ },+ {+ name: "literal tab inside string value",+ input: "{\"v\": \"a\tb\"}",+ want: `{"v": "a\tb"}`,+ },+ {+ name: "literal backspace and form feed inside string value",+ input: "{\"v\": \"a\x08\x0Cb\"}",+ want: `{"v": "a\b\fb"}`,+ },+ {+ name: `other control character (SOH) gets \uXXXX encoding`,+ input: "{\"v\": \"a\x01b\"}",+ want: `{"v": "a\u0001b"}`,+ },+ {+ name: "control character outside string - not touched",+ input: "{\n\"k\": \"v\"}",+ want: "{\n\"k\": \"v\"}",+ },+ {+ name: "multiple string fields, only affected one fixed",+ input: "{\"a\": \"ok\", \"b\": \"bad\nval\"}",+ want: `{"a": "ok", "b": "bad\nval"}`,+ },+ {+ name: "escaped backslash before quote not confused with string end",+ input: "{\"v\": \"path\\\\dir\"}",+ want: `{"v": "path\\dir"}`,+ },+ {+ name: "real-world vLLM crash case: index.html newline in arguments",+ input: "{\"input\": \"index.html] =\ncurl -s http://example.com\", \"cwd\": \"/work\"}",+ want: `{"input": "index.html] =\ncurl -s http://example.com", "cwd": "/work"}`,+ },+ {+ name: "empty string",+ input: "",+ want: "",+ },+ {+ name: "plain invalid JSON without control chars - returned as-is",+ input: `{broken`,+ want: `{broken`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := SanitizeJSONControlChars(tt.input)+ assert.Equal(t, tt.want, got)+ })+ }+}++func TestSanitizeToolCallArguments(t *testing.T) {+ makeChain := func(args string) []llms.MessageContent {+ return []llms.MessageContent{+ {+ Role: llms.ChatMessageTypeSystem,+ Parts: []llms.ContentPart{llms.TextContent{Text: "system"}},+ },+ {+ Role: llms.ChatMessageTypeHuman,+ Parts: []llms.ContentPart{llms.TextContent{Text: "do it"}},+ },+ {+ Role: llms.ChatMessageTypeAI,+ Parts: []llms.ContentPart{+ llms.ToolCall{+ ID: "call_abc",+ Type: "function",+ FunctionCall: &llms.FunctionCall{+ Name: "terminal",+ Arguments: args,+ },+ },+ },+ },+ {+ Role: llms.ChatMessageTypeTool,+ Parts: []llms.ContentPart{+ llms.ToolCallResponse{ToolCallID: "call_abc", Name: "terminal", Content: "ok"},+ },+ },+ }+ }++ getArgs := func(ast *ChainAST) string {+ for _, section := range ast.Sections {+ for _, pair := range section.Body {+ if pair.AIMessage == nil {+ continue+ }+ for _, part := range pair.AIMessage.Parts {+ if tc, ok := part.(llms.ToolCall); ok && tc.FunctionCall != nil {+ return tc.FunctionCall.Arguments+ }+ }+ }+ }+ return ""+ }++ tests := []struct {+ name string+ args string+ wantArgs string+ }{+ {+ name: "valid JSON - unchanged",+ args: `{"input": "ls -la", "cwd": "/work"}`,+ wantArgs: `{"input": "ls -la", "cwd": "/work"}`,+ },+ {+ name: "literal newline in argument value - gets escaped",+ args: "{\"input\": \"curl -s http://example.com\ncurl -s http://other.com\", \"cwd\": \"/work\"}",+ wantArgs: `{"input": "curl -s http://example.com\ncurl -s http://other.com", "cwd": "/work"}`,+ },+ {+ name: "multiple control chars - all escaped",+ args: "{\"input\": \"a\nb\rc\", \"cwd\": \"/work\"}",+ wantArgs: `{"input": "a\nb\rc", "cwd": "/work"}`,+ },+ {+ name: "already escaped newline in value - unchanged",+ args: `{"input": "line1\nline2", "cwd": "/work"}`,+ wantArgs: `{"input": "line1\nline2", "cwd": "/work"}`,+ },+ {+ name: "nil FunctionCall tool call does not panic",+ args: `{}`,+ wantArgs: `{}`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ chain := makeChain(tt.args)+ ast, err := NewChainAST(chain, false)+ assert.NoError(t, err)++ ast.SanitizeToolCallArguments()++ assert.Equal(t, tt.wantArgs, getArgs(ast))+ })+ }+}diff --git a/backend/pkg/providers/helpers.go b/backend/pkg/providers/helpers.goindex 7c714473..e040b953 100644--- a/backend/pkg/providers/helpers.go+++ b/backend/pkg/providers/helpers.go@@ -523,6 +523,8 @@ func (fp *flowProvider) restoreChain(return wrapErrorWithEvent("failed to normalize tool call IDs", err)}+ ast.SanitizeToolCallArguments()+if err := ast.ClearReasoning(); err != nil {return wrapErrorWithEvent("failed to clear reasoning", err)}@@ -631,6 +633,8 @@ func (fp *flowProvider) processChain(logger.WithError(err).Warn("failed to normalize tool call IDs")}+ ast.SanitizeToolCallArguments()+// Clear provider-specific reasoning signaturesif err := ast.ClearReasoning(); err != nil {logger.WithError(err).Warn("failed to clear reasoning")diff --git a/backend/pkg/providers/performer.go b/backend/pkg/providers/performer.goindex 44676fc0..32f8e462 100644--- a/backend/pkg/providers/performer.go+++ b/backend/pkg/providers/performer.go@@ -444,6 +444,7 @@ func (fp *flowProvider) callWithRetries(if toolCall.FunctionCall == nil {continue}+ toolCall.FunctionCall.Arguments = cast.SanitizeJSONControlChars(toolCall.FunctionCall.Arguments)result.funcCalls = append(result.funcCalls, toolCall)}}