Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
perf(evidence-receipts): read only the file tail for the previous hash
Sergey Kozyrenko4d25cf5c512f
diff --git a/backend/pkg/tools/evidence_receipts.go b/backend/pkg/tools/evidence_receipts.goindex cca340e8..453036ca 100644--- a/backend/pkg/tools/evidence_receipts.go+++ b/backend/pkg/tools/evidence_receipts.go@@ -1,7 +1,6 @@package toolsimport (- "bufio""bytes""context""crypto/sha256"@@ -227,53 +226,73 @@ func evidenceReceiptPathLock(path string) *sync.Mutex {}func readLastEvidenceReceiptHash(path string) (string, error) {+ line, err := readLastEvidenceReceiptLine(path)+ if err != nil {+ return "", err+ }+ if line == nil {+ return "", nil+ }++ var receipt evidenceReceipt+ if err := json.Unmarshal(line, &receipt); err != nil {+ return "", fmt.Errorf("failed to parse last evidence receipt: %w", err)+ }+ if receipt.Schema != evidenceReceiptSchema || receipt.Version != evidenceReceiptVersion {+ return "", fmt.Errorf("unsupported evidence receipt schema or version on last line")+ }++ hash, err := computeEvidenceReceiptHash(receipt)+ if err != nil {+ return "", fmt.Errorf("failed to hash last evidence receipt: %w", err)+ }+ if receipt.ReceiptHash != hash {+ return "", fmt.Errorf("evidence receipt hash mismatch on last line")+ }++ return receipt.ReceiptHash, nil+}++func readLastEvidenceReceiptLine(path string) ([]byte, error) {file, err := os.Open(path)if os.IsNotExist(err) {- return "", nil+ return nil, nil}if err != nil {- return "", fmt.Errorf("failed to read evidence receipt file: %w", err)+ return nil, fmt.Errorf("failed to read evidence receipt file: %w", err)}defer file.Close()- scanner := bufio.NewScanner(file)- scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)-- previousHash := ""- lineNumber := 0- for scanner.Scan() {- line := bytes.TrimSpace(scanner.Bytes())- if len(line) == 0 {- continue- }- lineNumber++-- var receipt evidenceReceipt- if err := json.Unmarshal(line, &receipt); err != nil {- return "", fmt.Errorf("failed to parse evidence receipt line %d: %w", lineNumber, err)- }- if receipt.Schema != evidenceReceiptSchema || receipt.Version != evidenceReceiptVersion {- return "", fmt.Errorf("unsupported evidence receipt schema or version on line %d", lineNumber)- }- if receipt.PreviousReceiptHash != previousHash {- return "", fmt.Errorf("evidence receipt chain mismatch on line %d", lineNumber)- }-- hash, err := computeEvidenceReceiptHash(receipt)- if err != nil {- return "", fmt.Errorf("failed to hash evidence receipt line %d: %w", lineNumber, err)- }- if receipt.ReceiptHash != hash {- return "", fmt.Errorf("evidence receipt hash mismatch on line %d", lineNumber)- }-- previousHash = receipt.ReceiptHash- }- if err := scanner.Err(); err != nil {- return "", fmt.Errorf("failed to scan evidence receipt file: %w", err)- }-- return previousHash, nil+ info, err := file.Stat()+ if err != nil {+ return nil, fmt.Errorf("failed to stat evidence receipt file: %w", err)+ }+ if info.Size() == 0 {+ return nil, nil+ }++ const maxReceiptLineWindow = 64 * 1024+ start := info.Size() - maxReceiptLineWindow+ if start < 0 {+ start = 0+ }++ window := make([]byte, info.Size()-start)+ if _, err := file.ReadAt(window, start); err != nil && err != io.EOF {+ return nil, fmt.Errorf("failed to read evidence receipt tail: %w", err)+ }++ trimmed := bytes.TrimRight(window, "\n")+ if len(trimmed) == 0 {+ return nil, nil+ }+ if idx := bytes.LastIndexByte(trimmed, '\n'); idx >= 0 {+ return trimmed[idx+1:], nil+ }+ if start > 0 {+ return nil, fmt.Errorf("last evidence receipt line exceeds %d bytes", maxReceiptLineWindow)+ }+ return trimmed, nil}func computeEvidenceReceiptHash(receipt evidenceReceipt) (string, error) {diff --git a/backend/pkg/tools/evidence_receipts_test.go b/backend/pkg/tools/evidence_receipts_test.goindex af4cbbe6..2b75c232 100644--- a/backend/pkg/tools/evidence_receipts_test.go+++ b/backend/pkg/tools/evidence_receipts_test.go@@ -406,6 +406,78 @@ func TestFileEvidenceReceiptRecorderKeepsChainsIntactWhenPathsShareAStripe(t *te}}+func TestReadLastEvidenceReceiptHashMatchesFullReadTail(t *testing.T) {+ t.Parallel()++ dir := t.TempDir()+ const flowID = int64(555)+ const receipts = 500++ recorder := newTestEvidenceReceiptRecorder(dir, flowID)+ for i := 0; i < receipts; i++ {+ if err := recorder.RecordFinished(t.Context(), testEvidenceReceiptEvent()); err != nil {+ t.Fatalf("RecordFinished() error: %v", err)+ }+ }++ path, err := evidenceReceiptsPath(dir, flowID)+ if err != nil {+ t.Fatalf("evidenceReceiptsPath() error: %v", err)+ }++ info, err := os.Stat(path)+ if err != nil {+ t.Fatalf("stat error: %v", err)+ }+ if info.Size() <= 64*1024 {+ t.Fatalf("file is %d bytes; need > 64KiB to exercise the windowed tail read", info.Size())+ }++ full := readEvidenceReceiptLines(t, path)+ want := full[len(full)-1].ReceiptHash++ got, err := readLastEvidenceReceiptHash(path)+ if err != nil {+ t.Fatalf("readLastEvidenceReceiptHash() error: %v", err)+ }+ if got != want {+ t.Fatalf("tail hash = %q, want %q (must equal the last line of a full read)", got, want)+ }+}++func TestReadLastEvidenceReceiptHashEmptyOrMissingFile(t *testing.T) {+ t.Parallel()++ dir := t.TempDir()+ path, err := evidenceReceiptsPath(dir, 556)+ if err != nil {+ t.Fatalf("evidenceReceiptsPath() error: %v", err)+ }++ got, err := readLastEvidenceReceiptHash(path)+ if err != nil {+ t.Fatalf("missing file: unexpected error: %v", err)+ }+ if got != "" {+ t.Fatalf("missing file tail = %q, want empty", got)+ }++ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {+ t.Fatalf("failed to create evidence dir: %v", err)+ }+ if err := os.WriteFile(path, nil, 0644); err != nil {+ t.Fatalf("failed to create empty file: %v", err)+ }++ got, err = readLastEvidenceReceiptHash(path)+ if err != nil {+ t.Fatalf("empty file: unexpected error: %v", err)+ }+ if got != "" {+ t.Fatalf("empty file tail = %q, want empty", got)+ }+}+func collidingStripeFlowIDs(t *testing.T, dir string) (int64, int64) {t.Helper()