Skip to content

Find, verify, and analyze leaked credentials

Add custom detectors configuration parsing (#927)

Miccah4409210b87f3
diff --git a/go.mod b/go.mod
index 1563caeed..74dbbf161 100644
--- a/go.mod
+++ b/go.mod
@@ -61,6 +61,7 @@ require (
google.golang.org/protobuf v1.28.1
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
+ sigs.k8s.io/yaml v1.3.0
)
require (
@@ -143,5 +144,6 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/grpc v1.50.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index feccf6a3a..451133881 100644
--- a/go.sum
+++ b/go.sum
@@ -861,3 +861,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go
new file mode 100644
index 000000000..96a160019
--- /dev/null
+++ b/pkg/custom_detectors/custom_detectors_test.go
@@ -0,0 +1,68 @@
+package custom_detectors
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
+)
+
+const testCustomRegexYaml = `name: Internal bi tool
+keywords:
+- secret_v1_
+- pat_v2_
+regex:
+ id_pat_example: ([a-zA-Z0-9]{32})
+ secret_pat_example: ([a-zA-Z0-9]{32})
+verify:
+- endpoint: http://localhost:8000/{id_pat_example}
+ unsafe: true
+ headers:
+ - 'Authorization: Bearer {secret_pat_example.0}'
+ successRanges:
+ - 200-250
+ - '288'`
+
+// Helper function to test equality to the data in testCustomRegexYaml.
+func assertExpected(t *testing.T, got *custom_detectorspb.CustomRegex) {
+ assert.Equal(t, "Internal bi tool", got.Name)
+ assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords)
+ assert.Equal(t, map[string]string{
+ "id_pat_example": "([a-zA-Z0-9]{32})",
+ "secret_pat_example": "([a-zA-Z0-9]{32})",
+ }, got.Regex)
+ assert.Equal(t, 1, len(got.Verify))
+ assert.Equal(t, "http://localhost:8000/{id_pat_example}", got.Verify[0].Endpoint)
+ assert.Equal(t, true, got.Verify[0].Unsafe)
+ assert.Equal(t, []string{"Authorization: Bearer {secret_pat_example.0}"}, got.Verify[0].Headers)
+ assert.Equal(t, []string{"200-250", "288"}, got.Verify[0].SuccessRanges)
+}
+
+func TestCustomRegexParsing(t *testing.T) {
+ var message custom_detectorspb.CustomRegex
+
+ assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testCustomRegexYaml), &message))
+ assertExpected(t, &message)
+}
+
+func TestCustomDetectorsParsing(t *testing.T) {
+ var testYamlConfig string
+ // Build a config file using testCustomRegexYaml.
+ {
+ var lines []string
+ for i, line := range strings.Split(testCustomRegexYaml, "\n") {
+ if i == 0 {
+ lines = append(lines, line)
+ continue
+ }
+ lines = append(lines, " "+line)
+ }
+ testYamlConfig = "detectors:\n- " + strings.Join(lines, "\n")
+ }
+
+ var messages custom_detectorspb.CustomDetectors
+ assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testYamlConfig), &messages))
+ assertExpected(t, messages.Detectors[0])
+}
diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.go
new file mode 100644
index 000000000..1d0a83fa3
--- /dev/null
+++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.go
@@ -0,0 +1,348 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.26.0
+// protoc v3.20.0
+// source: custom_detectors.proto
+
+package custom_detectorspb
+
+import (
+ _ "github.com/envoyproxy/protoc-gen-validate/validate"
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type CustomDetectors struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Detectors []*CustomRegex `protobuf:"bytes,1,rep,name=detectors,proto3" json:"detectors,omitempty"`
+}
+
+func (x *CustomDetectors) Reset() {
+ *x = CustomDetectors{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_custom_detectors_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *CustomDetectors) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CustomDetectors) ProtoMessage() {}
+
+func (x *CustomDetectors) ProtoReflect() protoreflect.Message {
+ mi := &file_custom_detectors_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CustomDetectors.ProtoReflect.Descriptor instead.
+func (*CustomDetectors) Descriptor() ([]byte, []int) {
+ return file_custom_detectors_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *CustomDetectors) GetDetectors() []*CustomRegex {
+ if x != nil {
+ return x.Detectors
+ }
+ return nil
+}
+
+type CustomRegex struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+ Keywords []string `protobuf:"bytes,2,rep,name=keywords,proto3" json:"keywords,omitempty"`
+ Regex map[string]string `protobuf:"bytes,3,rep,name=regex,proto3" json:"regex,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ Verify []*VerifierConfig `protobuf:"bytes,4,rep,name=verify,proto3" json:"verify,omitempty"`
+}
+
+func (x *CustomRegex) Reset() {
+ *x = CustomRegex{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_custom_detectors_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *CustomRegex) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CustomRegex) ProtoMessage() {}
+
+func (x *CustomRegex) ProtoReflect() protoreflect.Message {
+ mi := &file_custom_detectors_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CustomRegex.ProtoReflect.Descriptor instead.
+func (*CustomRegex) Descriptor() ([]byte, []int) {
+ return file_custom_detectors_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *CustomRegex) GetName() string {
+ if x != nil {
+ return x.Name
+ }
+ return ""
+}
+
+func (x *CustomRegex) GetKeywords() []string {
+ if x != nil {
+ return x.Keywords
+ }
+ return nil
+}
+
+func (x *CustomRegex) GetRegex() map[string]string {
+ if x != nil {
+ return x.Regex
+ }
+ return nil
+}
+
+func (x *CustomRegex) GetVerify() []*VerifierConfig {
+ if x != nil {
+ return x.Verify
+ }
+ return nil
+}
+
+type VerifierConfig struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
+ Unsafe bool `protobuf:"varint,2,opt,name=unsafe,proto3" json:"unsafe,omitempty"`
+ Headers []string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty"`
+ SuccessRanges []string `protobuf:"bytes,4,rep,name=successRanges,proto3" json:"successRanges,omitempty"`
+}
+
+func (x *VerifierConfig) Reset() {
+ *x = VerifierConfig{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_custom_detectors_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *VerifierConfig) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*VerifierConfig) ProtoMessage() {}
+
+func (x *VerifierConfig) ProtoReflect() protoreflect.Message {
+ mi := &file_custom_detectors_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use VerifierConfig.ProtoReflect.Descriptor instead.
+func (*VerifierConfig) Descriptor() ([]byte, []int) {
+ return file_custom_detectors_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *VerifierConfig) GetEndpoint() string {
+ if x != nil {
+ return x.Endpoint
+ }
+ return ""
+}
+
+func (x *VerifierConfig) GetUnsafe() bool {
+ if x != nil {
+ return x.Unsafe
+ }
+ return false
+}
+
+func (x *VerifierConfig) GetHeaders() []string {
+ if x != nil {
+ return x.Headers
+ }
+ return nil
+}
+
+func (x *VerifierConfig) GetSuccessRanges() []string {
+ if x != nil {
+ return x.SuccessRanges
+ }
+ return nil
+}
+
+var File_custom_detectors_proto protoreflect.FileDescriptor
+
+var file_custom_detectors_proto_rawDesc = []byte{
+ 0x0a, 0x16, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
+ 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d,
+ 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69,
+ 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72,
+ 0x6f, 0x74, 0x6f, 0x22, 0x4e, 0x0a, 0x0f, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x44, 0x65, 0x74,
+ 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
+ 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74,
+ 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73,
+ 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
+ 0x6f, 0x72, 0x73, 0x22, 0xf1, 0x01, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65,
+ 0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+ 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
+ 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
+ 0x72, 0x64, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x03,
+ 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65,
+ 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65,
+ 0x78, 0x2e, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x65,
+ 0x67, 0x65, 0x78, 0x12, 0x38, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18, 0x04, 0x20,
+ 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74,
+ 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43,
+ 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x1a, 0x38, 0x0a,
+ 0x0a, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+ 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
+ 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61,
+ 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69,
+ 0x66, 0x69, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
+ 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
+ 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+ 0x12, 0x16, 0x0a, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
+ 0x52, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64,
+ 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65,
+ 0x72, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e,
+ 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65,
+ 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68,
+ 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+ 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+ 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74,
+ 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06,
+ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+ file_custom_detectors_proto_rawDescOnce sync.Once
+ file_custom_detectors_proto_rawDescData = file_custom_detectors_proto_rawDesc
+)
+
+func file_custom_detectors_proto_rawDescGZIP() []byte {
+ file_custom_detectors_proto_rawDescOnce.Do(func() {
+ file_custom_detectors_proto_rawDescData = protoimpl.X.CompressGZIP(file_custom_detectors_proto_rawDescData)
+ })
+ return file_custom_detectors_proto_rawDescData
+}
+
+var file_custom_detectors_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_custom_detectors_proto_goTypes = []interface{}{
+ (*CustomDetectors)(nil), // 0: custom_detectors.CustomDetectors
+ (*CustomRegex)(nil), // 1: custom_detectors.CustomRegex
+ (*VerifierConfig)(nil), // 2: custom_detectors.VerifierConfig
+ nil, // 3: custom_detectors.CustomRegex.RegexEntry
+}
+var file_custom_detectors_proto_depIdxs = []int32{
+ 1, // 0: custom_detectors.CustomDetectors.detectors:type_name -> custom_detectors.CustomRegex
+ 3, // 1: custom_detectors.CustomRegex.regex:type_name -> custom_detectors.CustomRegex.RegexEntry
+ 2, // 2: custom_detectors.CustomRegex.verify:type_name -> custom_detectors.VerifierConfig
+ 3, // [3:3] is the sub-list for method output_type
+ 3, // [3:3] is the sub-list for method input_type
+ 3, // [3:3] is the sub-list for extension type_name
+ 3, // [3:3] is the sub-list for extension extendee
+ 0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_custom_detectors_proto_init() }
+func file_custom_detectors_proto_init() {
+ if File_custom_detectors_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_custom_detectors_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*CustomDetectors); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_custom_detectors_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*CustomRegex); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_custom_detectors_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*VerifierConfig); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_custom_detectors_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 4,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_custom_detectors_proto_goTypes,
+ DependencyIndexes: file_custom_detectors_proto_depIdxs,
+ MessageInfos: file_custom_detectors_proto_msgTypes,
+ }.Build()
+ File_custom_detectors_proto = out.File
+ file_custom_detectors_proto_rawDesc = nil
+ file_custom_detectors_proto_goTypes = nil
+ file_custom_detectors_proto_depIdxs = nil
+}
diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go
new file mode 100644
index 000000000..2b87120b3
--- /dev/null
+++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go
@@ -0,0 +1,421 @@
+// Code generated by protoc-gen-validate. DO NOT EDIT.
+// source: custom_detectors.proto
+
+package custom_detectorspb
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "net"
+ "net/mail"
+ "net/url"
+ "regexp"
+ "sort"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "google.golang.org/protobuf/types/known/anypb"
+)
+
+// ensure the imports are used
+var (
+ _ = bytes.MinRead
+ _ = errors.New("")
+ _ = fmt.Print
+ _ = utf8.UTFMax
+ _ = (*regexp.Regexp)(nil)
+ _ = (*strings.Reader)(nil)
+ _ = net.IPv4len
+ _ = time.Duration(0)
+ _ = (*url.URL)(nil)
+ _ = (*mail.Address)(nil)
+ _ = anypb.Any{}
+ _ = sort.Sort
+)
+
+// Validate checks the field values on CustomDetectors with the rules defined
+// in the proto definition for this message. If any rules are violated, the
+// first error encountered is returned, or nil if there are no violations.
+func (m *CustomDetectors) Validate() error {
+ return m.validate(false)
+}
+
+// ValidateAll checks the field values on CustomDetectors with the rules
+// defined in the proto definition for this message. If any rules are
+// violated, the result is a list of violation errors wrapped in
+// CustomDetectorsMultiError, or nil if none found.
+func (m *CustomDetectors) ValidateAll() error {
+ return m.validate(true)
+}
+
+func (m *CustomDetectors) validate(all bool) error {
+ if m == nil {
+ return nil
+ }
+
+ var errors []error
+
+ for idx, item := range m.GetDetectors() {
+ _, _ = idx, item
+
+ if all {
+ switch v := interface{}(item).(type) {
+ case interface{ ValidateAll() error }:
+ if err := v.ValidateAll(); err != nil {
+ errors = append(errors, CustomDetectorsValidationError{
+ field: fmt.Sprintf("Detectors[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ })
+ }
+ case interface{ Validate() error }:
+ if err := v.Validate(); err != nil {
+ errors = append(errors, CustomDetectorsValidationError{
+ field: fmt.Sprintf("Detectors[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ })
+ }
+ }
+ } else if v, ok := interface{}(item).(interface{ Validate() error }); ok {
+ if err := v.Validate(); err != nil {
+ return CustomDetectorsValidationError{
+ field: fmt.Sprintf("Detectors[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ }
+ }
+ }
+
+ }
+
+ if len(errors) > 0 {
+ return CustomDetectorsMultiError(errors)
+ }
+
+ return nil
+}
+
+// CustomDetectorsMultiError is an error wrapping multiple validation errors
+// returned by CustomDetectors.ValidateAll() if the designated constraints
+// aren't met.
+type CustomDetectorsMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m CustomDetectorsMultiError) Error() string {
+ var msgs []string
+ for _, err := range m {
+ msgs = append(msgs, err.Error())
+ }
+ return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m CustomDetectorsMultiError) AllErrors() []error { return m }
+
+// CustomDetectorsValidationError is the validation error returned by
+// CustomDetectors.Validate if the designated constraints aren't met.
+type CustomDetectorsValidationError struct {
+ field string
+ reason string
+ cause error
+ key bool
+}
+
+// Field function returns field value.
+func (e CustomDetectorsValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e CustomDetectorsValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e CustomDetectorsValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e CustomDetectorsValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e CustomDetectorsValidationError) ErrorName() string { return "CustomDetectorsValidationError" }
+
+// Error satisfies the builtin error interface
+func (e CustomDetectorsValidationError) Error() string {
+ cause := ""
+ if e.cause != nil {
+ cause = fmt.Sprintf(" | caused by: %v", e.cause)
+ }
+
+ key := ""
+ if e.key {
+ key = "key for "
+ }
+
+ return fmt.Sprintf(
+ "invalid %sCustomDetectors.%s: %s%s",
+ key,
+ e.field,
+ e.reason,
+ cause)
+}
+
+var _ error = CustomDetectorsValidationError{}
+
+var _ interface {
+ Field() string
+ Reason() string
+ Key() bool
+ Cause() error
+ ErrorName() string
+} = CustomDetectorsValidationError{}
+
+// Validate checks the field values on CustomRegex with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *CustomRegex) Validate() error {
+ return m.validate(false)
+}
+
+// ValidateAll checks the field values on CustomRegex with the rules defined in
+// the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in CustomRegexMultiError, or
+// nil if none found.
+func (m *CustomRegex) ValidateAll() error {
+ return m.validate(true)
+}
+
+func (m *CustomRegex) validate(all bool) error {
+ if m == nil {
+ return nil
+ }
+
+ var errors []error
+
+ // no validation rules for Name
+
+ // no validation rules for Regex
+
+ for idx, item := range m.GetVerify() {
+ _, _ = idx, item
+
+ if all {
+ switch v := interface{}(item).(type) {
+ case interface{ ValidateAll() error }:
+ if err := v.ValidateAll(); err != nil {
+ errors = append(errors, CustomRegexValidationError{
+ field: fmt.Sprintf("Verify[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ })
+ }
+ case interface{ Validate() error }:
+ if err := v.Validate(); err != nil {
+ errors = append(errors, CustomRegexValidationError{
+ field: fmt.Sprintf("Verify[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ })
+ }
+ }
+ } else if v, ok := interface{}(item).(interface{ Validate() error }); ok {
+ if err := v.Validate(); err != nil {
+ return CustomRegexValidationError{
+ field: fmt.Sprintf("Verify[%v]", idx),
+ reason: "embedded message failed validation",
+ cause: err,
+ }
+ }
+ }
+
+ }
+
+ if len(errors) > 0 {
+ return CustomRegexMultiError(errors)
+ }
+
+ return nil
+}
+
+// CustomRegexMultiError is an error wrapping multiple validation errors
+// returned by CustomRegex.ValidateAll() if the designated constraints aren't met.
+type CustomRegexMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m CustomRegexMultiError) Error() string {
+ var msgs []string
+ for _, err := range m {
+ msgs = append(msgs, err.Error())
+ }
+ return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m CustomRegexMultiError) AllErrors() []error { return m }
+
+// CustomRegexValidationError is the validation error returned by
+// CustomRegex.Validate if the designated constraints aren't met.
+type CustomRegexValidationError struct {
+ field string
+ reason string
+ cause error
+ key bool
+}
+
+// Field function returns field value.
+func (e CustomRegexValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e CustomRegexValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e CustomRegexValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e CustomRegexValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e CustomRegexValidationError) ErrorName() string { return "CustomRegexValidationError" }
+
+// Error satisfies the builtin error interface
+func (e CustomRegexValidationError) Error() string {
+ cause := ""
+ if e.cause != nil {
+ cause = fmt.Sprintf(" | caused by: %v", e.cause)
+ }
+
+ key := ""
+ if e.key {
+ key = "key for "
+ }
+
+ return fmt.Sprintf(
+ "invalid %sCustomRegex.%s: %s%s",
+ key,
+ e.field,
+ e.reason,
+ cause)
+}
+
+var _ error = CustomRegexValidationError{}
+
+var _ interface {
+ Field() string
+ Reason() string
+ Key() bool
+ Cause() error
+ ErrorName() string
+} = CustomRegexValidationError{}
+
+// Validate checks the field values on VerifierConfig with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *VerifierConfig) Validate() error {
+ return m.validate(false)
+}
+
+// ValidateAll checks the field values on VerifierConfig with the rules defined
+// in the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in VerifierConfigMultiError,
+// or nil if none found.
+func (m *VerifierConfig) ValidateAll() error {
+ return m.validate(true)
+}
+
+func (m *VerifierConfig) validate(all bool) error {
+ if m == nil {
+ return nil
+ }
+
+ var errors []error
+
+ if _, err := url.Parse(m.GetEndpoint()); err != nil {
+ err = VerifierConfigValidationError{
+ field: "Endpoint",
+ reason: "value must be a valid URI",
+ cause: err,
+ }
+ if !all {
+ return err
+ }
+ errors = append(errors, err)
+ }
+
+ // no validation rules for Unsafe
+
+ if len(errors) > 0 {
+ return VerifierConfigMultiError(errors)
+ }
+
+ return nil
+}
+
+// VerifierConfigMultiError is an error wrapping multiple validation errors
+// returned by VerifierConfig.ValidateAll() if the designated constraints
+// aren't met.
+type VerifierConfigMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m VerifierConfigMultiError) Error() string {
+ var msgs []string
+ for _, err := range m {
+ msgs = append(msgs, err.Error())
+ }
+ return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m VerifierConfigMultiError) AllErrors() []error { return m }
+
+// VerifierConfigValidationError is the validation error returned by
+// VerifierConfig.Validate if the designated constraints aren't met.
+type VerifierConfigValidationError struct {
+ field string
+ reason string
+ cause error
+ key bool
+}
+
+// Field function returns field value.
+func (e VerifierConfigValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e VerifierConfigValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e VerifierConfigValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e VerifierConfigValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e VerifierConfigValidationError) ErrorName() string { return "VerifierConfigValidationError" }
+
+// Error satisfies the builtin error interface
+func (e VerifierConfigValidationError) Error() string {
+ cause := ""
+ if e.cause != nil {
+ cause = fmt.Sprintf(" | caused by: %v", e.cause)
+ }
+
+ key := ""
+ if e.key {
+ key = "key for "
+ }
+
+ return fmt.Sprintf(
+ "invalid %sVerifierConfig.%s: %s%s",
+ key,
+ e.field,
+ e.reason,
+ cause)
+}
+
+var _ error = VerifierConfigValidationError{}
+
+var _ interface {
+ Field() string
+ Reason() string
+ Key() bool
+ Cause() error
+ ErrorName() string
+} = VerifierConfigValidationError{}
diff --git a/pkg/protoyaml/protoyaml.go b/pkg/protoyaml/protoyaml.go
new file mode 100644
index 000000000..7fe0c24bb
--- /dev/null
+++ b/pkg/protoyaml/protoyaml.go
@@ -0,0 +1,39 @@
+package protoyaml
+
+import (
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "sigs.k8s.io/yaml"
+)
+
+var nonStrict = protojson.UnmarshalOptions{DiscardUnknown: true}
+var strict = protojson.UnmarshalOptions{DiscardUnknown: false}
+
+// Marshal writes the given proto.Message in YAML format.
+func Marshal(m proto.Message) ([]byte, error) {
+ json, err := protojson.Marshal(m)
+ if err != nil {
+ return nil, err
+ }
+ return yaml.JSONToYAML(json)
+}
+
+// Unmarshal reads the given []byte into the given proto.Message, discarding
+// any unknown fields in the input.
+func Unmarshal(b []byte, m proto.Message) error {
+ json, err := yaml.YAMLToJSON(b)
+ if err != nil {
+ return err
+ }
+ return nonStrict.Unmarshal(json, m)
+}
+
+// UnmarshalStrict reads the given []byte into the given proto.Message. If there
+// are any unknown fields in the input, an error is returned.
+func UnmarshalStrict(b []byte, m proto.Message) error {
+ json, err := yaml.YAMLToJSON(b)
+ if err != nil {
+ return err
+ }
+ return strict.Unmarshal(json, m)
+}
diff --git a/proto/custom_detectors.proto b/proto/custom_detectors.proto
new file mode 100644
index 000000000..04ef1ebb9
--- /dev/null
+++ b/proto/custom_detectors.proto
@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package custom_detectors;
+
+option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb";
+
+import "validate/validate.proto";
+
+message CustomDetectors {
+ repeated CustomRegex detectors = 1;
+}
+
+message CustomRegex {
+ string name = 1;
+ repeated string keywords = 2;
+ map<string, string> regex = 3;
+ repeated VerifierConfig verify = 4;
+}
+
+message VerifierConfig {
+ string endpoint = 1 [(validate.rules).string.uri_ref = true];
+ bool unsafe = 2;
+ repeated string headers = 3;
+ repeated string successRanges = 4;
+}
diff --git a/scripts/gen_proto.sh b/scripts/gen_proto.sh
index 226754cdf..d3df1c128 100755
--- a/scripts/gen_proto.sh
+++ b/scripts/gen_proto.sh
@@ -30,3 +30,10 @@ protoc -I proto/ \
--go_out=plugins=grpc:./pkg/pb/source_metadatapb --go_opt=paths=source_relative \
--validate_out="lang=go,paths=source_relative:./pkg/pb/source_metadatapb" \
proto/source_metadata.proto
+protoc -I proto/ \
+ -I ${GOPATH}/src \
+ -I /usr/local/include \
+ -I ${GOPATH}/src/github.com/envoyproxy/protoc-gen-validate \
+ --go_out=plugins=grpc:./pkg/pb/custom_detectorspb --go_opt=paths=source_relative \
+ --validate_out="lang=go,paths=source_relative:./pkg/pb/custom_detectorspb" \
+ proto/custom_detectors.proto