Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

feat(cast): add JSON control character sanitization for tool call arguments

Dmitry Ngf57e988586a5
diff --git a/backend/pkg/cast/chain_ast.go b/backend/pkg/cast/chain_ast.go
index 7445de29..cddbccc9 100644
--- a/backend/pkg/cast/chain_ast.go
+++ b/backend/pkg/cast/chain_ast.go
@@ -1,6 +1,7 @@
package cast
import (
+ "encoding/json"
"fmt"
"pentagi/pkg/templates"
"sort"
@@ -1056,6 +1057,104 @@ func ContainsToolCallReasoning(messages []llms.MessageContent) bool {
return false
}
+// SanitizeJSONControlChars escapes any literal control characters (0x00–0x1F, 0x7F) that
+// appear inside JSON string values. Such characters are disallowed by the JSON spec and
+// must be encoded as \n, \r, \t, or \uXXXX escape sequences.
+//
+// The scanner is context-aware: it tracks whether it is currently inside a JSON string and
+// skips over already-escaped sequences so they are never double-escaped.
+func SanitizeJSONControlChars(s string) string {
+ if json.Valid([]byte(s)) {
+ return s
+ }
+
+ var buf strings.Builder
+ buf.Grow(len(s) + 32)
+
+ inString := false
+ escaped := false
+
+ for i := 0; i < len(s); {
+ b := s[i]
+
+ if escaped {
+ buf.WriteByte(b)
+ i++
+ escaped = false
+ continue
+ }
+
+ if inString && b == '\\' {
+ buf.WriteByte(b)
+ i++
+ escaped = true
+ continue
+ }
+
+ if b == '"' {
+ buf.WriteByte(b)
+ i++
+ inString = !inString
+ continue
+ }
+
+ if inString && b < 0x20 {
+ switch b {
+ case '\n':
+ buf.WriteString(`\n`)
+ case '\r':
+ buf.WriteString(`\r`)
+ case '\t':
+ buf.WriteString(`\t`)
+ case '\b':
+ buf.WriteString(`\b`)
+ case '\f':
+ buf.WriteString(`\f`)
+ default:
+ fmt.Fprintf(&buf, `\u%04x`, b)
+ }
+ i++
+ continue
+ }
+
+ buf.WriteByte(b)
+ i++
+ }
+
+ return buf.String()
+}
+
+// SanitizeToolCallArguments scans every tool call in the chain and ensures its Arguments
+// field is valid JSON. Literal control characters that some LLM providers occasionally
+// emit inside string values are escaped so that downstream consumers (e.g. vLLM's
+// _postprocess_messages) can parse the arguments without errors.
+func (ast *ChainAST) SanitizeToolCallArguments() {
+ for _, section := range ast.Sections {
+ for _, bodyPair := range section.Body {
+ if bodyPair.Type != RequestResponse && bodyPair.Type != Summarization {
+ continue
+ }
+
+ if bodyPair.AIMessage == nil {
+ continue
+ }
+
+ for pdx, part := range bodyPair.AIMessage.Parts {
+ toolCall, ok := part.(llms.ToolCall)
+ if !ok || toolCall.FunctionCall == nil {
+ continue
+ }
+
+ sanitized := SanitizeJSONControlChars(toolCall.FunctionCall.Arguments)
+ if sanitized != toolCall.FunctionCall.Arguments {
+ toolCall.FunctionCall.Arguments = sanitized
+ bodyPair.AIMessage.Parts[pdx] = toolCall
+ }
+ }
+ }
+ }
+}
+
// ExtractReasoningMessage extracts the first AI message that contains reasoning content
// in a TextContent part. This is useful for preserving reasoning messages when summarizing
// content for providers like Kimi (Moonshot) that require reasoning_content before ToolCall.
diff --git a/backend/pkg/cast/chain_ast_test.go b/backend/pkg/cast/chain_ast_test.go
index 28f4ee51..edfe4fb0 100644
--- a/backend/pkg/cast/chain_ast_test.go
+++ b/backend/pkg/cast/chain_ast_test.go
@@ -3015,3 +3015,178 @@ func TestClearReasoning_IntegrationWithNormalize(t *testing.T) {
t.Log("Successfully normalized IDs and cleared reasoning for provider switch")
}
+
+func TestSanitizeJSONControlChars(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ want string
+ }{
+ {
+ name: "already valid JSON - no changes",
+ input: `{"input": "hello world"}`,
+ want: `{"input": "hello world"}`,
+ },
+ {
+ name: "already valid JSON with escaped newline - no changes",
+ input: `{"input": "line1\nline2"}`,
+ want: `{"input": "line1\nline2"}`,
+ },
+ {
+ name: "literal newline inside string value",
+ input: "{\"input\": \"line1\nline2\"}",
+ want: `{"input": "line1\nline2"}`,
+ },
+ {
+ name: "literal carriage return inside string value",
+ input: "{\"cmd\": \"echo\rtest\"}",
+ want: `{"cmd": "echo\rtest"}`,
+ },
+ {
+ name: "literal tab inside string value",
+ input: "{\"v\": \"a\tb\"}",
+ want: `{"v": "a\tb"}`,
+ },
+ {
+ name: "literal backspace and form feed inside string value",
+ input: "{\"v\": \"a\x08\x0Cb\"}",
+ want: `{"v": "a\b\fb"}`,
+ },
+ {
+ name: `other control character (SOH) gets \uXXXX encoding`,
+ input: "{\"v\": \"a\x01b\"}",
+ want: `{"v": "a\u0001b"}`,
+ },
+ {
+ name: "control character outside string - not touched",
+ input: "{\n\"k\": \"v\"}",
+ want: "{\n\"k\": \"v\"}",
+ },
+ {
+ name: "multiple string fields, only affected one fixed",
+ input: "{\"a\": \"ok\", \"b\": \"bad\nval\"}",
+ want: `{"a": "ok", "b": "bad\nval"}`,
+ },
+ {
+ name: "escaped backslash before quote not confused with string end",
+ input: "{\"v\": \"path\\\\dir\"}",
+ want: `{"v": "path\\dir"}`,
+ },
+ {
+ name: "real-world vLLM crash case: index.html newline in arguments",
+ input: "{\"input\": \"index.html] =\ncurl -s http://example.com\", \"cwd\": \"/work\"}",
+ want: `{"input": "index.html] =\ncurl -s http://example.com", "cwd": "/work"}`,
+ },
+ {
+ name: "empty string",
+ input: "",
+ want: "",
+ },
+ {
+ name: "plain invalid JSON without control chars - returned as-is",
+ input: `{broken`,
+ want: `{broken`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := SanitizeJSONControlChars(tt.input)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestSanitizeToolCallArguments(t *testing.T) {
+ makeChain := func(args string) []llms.MessageContent {
+ return []llms.MessageContent{
+ {
+ Role: llms.ChatMessageTypeSystem,
+ Parts: []llms.ContentPart{llms.TextContent{Text: "system"}},
+ },
+ {
+ Role: llms.ChatMessageTypeHuman,
+ Parts: []llms.ContentPart{llms.TextContent{Text: "do it"}},
+ },
+ {
+ Role: llms.ChatMessageTypeAI,
+ Parts: []llms.ContentPart{
+ llms.ToolCall{
+ ID: "call_abc",
+ Type: "function",
+ FunctionCall: &llms.FunctionCall{
+ Name: "terminal",
+ Arguments: args,
+ },
+ },
+ },
+ },
+ {
+ Role: llms.ChatMessageTypeTool,
+ Parts: []llms.ContentPart{
+ llms.ToolCallResponse{ToolCallID: "call_abc", Name: "terminal", Content: "ok"},
+ },
+ },
+ }
+ }
+
+ getArgs := func(ast *ChainAST) string {
+ for _, section := range ast.Sections {
+ for _, pair := range section.Body {
+ if pair.AIMessage == nil {
+ continue
+ }
+ for _, part := range pair.AIMessage.Parts {
+ if tc, ok := part.(llms.ToolCall); ok && tc.FunctionCall != nil {
+ return tc.FunctionCall.Arguments
+ }
+ }
+ }
+ }
+ return ""
+ }
+
+ tests := []struct {
+ name string
+ args string
+ wantArgs string
+ }{
+ {
+ name: "valid JSON - unchanged",
+ args: `{"input": "ls -la", "cwd": "/work"}`,
+ wantArgs: `{"input": "ls -la", "cwd": "/work"}`,
+ },
+ {
+ name: "literal newline in argument value - gets escaped",
+ args: "{\"input\": \"curl -s http://example.com\ncurl -s http://other.com\", \"cwd\": \"/work\"}",
+ wantArgs: `{"input": "curl -s http://example.com\ncurl -s http://other.com", "cwd": "/work"}`,
+ },
+ {
+ name: "multiple control chars - all escaped",
+ args: "{\"input\": \"a\nb\rc\", \"cwd\": \"/work\"}",
+ wantArgs: `{"input": "a\nb\rc", "cwd": "/work"}`,
+ },
+ {
+ name: "already escaped newline in value - unchanged",
+ args: `{"input": "line1\nline2", "cwd": "/work"}`,
+ wantArgs: `{"input": "line1\nline2", "cwd": "/work"}`,
+ },
+ {
+ name: "nil FunctionCall tool call does not panic",
+ args: `{}`,
+ wantArgs: `{}`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ chain := makeChain(tt.args)
+ ast, err := NewChainAST(chain, false)
+ assert.NoError(t, err)
+
+ ast.SanitizeToolCallArguments()
+
+ assert.Equal(t, tt.wantArgs, getArgs(ast))
+ })
+ }
+}
diff --git a/backend/pkg/providers/helpers.go b/backend/pkg/providers/helpers.go
index 7c714473..e040b953 100644
--- a/backend/pkg/providers/helpers.go
+++ b/backend/pkg/providers/helpers.go
@@ -523,6 +523,8 @@ func (fp *flowProvider) restoreChain(
return wrapErrorWithEvent("failed to normalize tool call IDs", err)
}
+ ast.SanitizeToolCallArguments()
+
if err := ast.ClearReasoning(); err != nil {
return wrapErrorWithEvent("failed to clear reasoning", err)
}
@@ -631,6 +633,8 @@ func (fp *flowProvider) processChain(
logger.WithError(err).Warn("failed to normalize tool call IDs")
}
+ ast.SanitizeToolCallArguments()
+
// Clear provider-specific reasoning signatures
if err := ast.ClearReasoning(); err != nil {
logger.WithError(err).Warn("failed to clear reasoning")
diff --git a/backend/pkg/providers/performer.go b/backend/pkg/providers/performer.go
index 44676fc0..32f8e462 100644
--- a/backend/pkg/providers/performer.go
+++ b/backend/pkg/providers/performer.go
@@ -444,6 +444,7 @@ func (fp *flowProvider) callWithRetries(
if toolCall.FunctionCall == nil {
continue
}
+ toolCall.FunctionCall.Arguments = cast.SanitizeJSONControlChars(toolCall.FunctionCall.Arguments)
result.funcCalls = append(result.funcCalls, toolCall)
}
}