Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add generic glob filter (#1858)
Miccah23ae970bb001
diff --git a/go.mod b/go.modindex 64029ba20..a09acc9b5 100644--- a/go.mod+++ b/go.mod@@ -81,6 +81,7 @@ require (google.golang.org/protobuf v1.31.0gopkg.in/alecthomas/kingpin.v2 v2.2.6gopkg.in/h2non/gock.v1 v1.1.2+ pgregory.net/rapid v1.1.0sigs.k8s.io/yaml v1.3.0)diff --git a/go.sum b/go.sumindex 7fc55883c..704cc19c5 100644--- a/go.sum+++ b/go.sum@@ -1013,6 +1013,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=diff --git a/pkg/common/glob/glob.go b/pkg/common/glob/glob.gonew file mode 100644index 000000000..a31bbe6fc--- /dev/null+++ b/pkg/common/glob/glob.go@@ -0,0 +1,120 @@+package common++import (+ "fmt"++ "github.com/gobwas/glob"+)++// Filter is a generic filter for excluding and including globs (limited+// regular expressions). Exclusion takes precedence if both include and exclude+// lists are provided.+type Filter struct {+ exclude []glob.Glob+ include []glob.Glob+}++type globFilterOpt func(*Filter) error++// WithExcludeGlobs adds exclude globs to the filter.+func WithExcludeGlobs(excludes ...string) globFilterOpt {+ return func(f *Filter) error {+ for _, exclude := range excludes {+ g, err := glob.Compile(exclude)+ if err != nil {+ return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)+ }+ f.exclude = append(f.exclude, g)+ }+ return nil+ }+}++// WithIncludeGlobs adds include globs to the filter.+func WithIncludeGlobs(includes ...string) globFilterOpt {+ return func(f *Filter) error {+ for _, include := range includes {+ g, err := glob.Compile(include)+ if err != nil {+ return fmt.Errorf("invalid include glob %q: %w", include, err)+ }+ f.include = append(f.include, g)+ }+ return nil+ }+}++// NewGlobFilter creates a new Filter with the provided options.+func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {+ filter := &Filter{}+ for _, opt := range opts {+ if err := opt(filter); err != nil {+ return nil, err+ }+ }+ return filter, nil+}++// ShouldInclude returns whether the object is in the include list or not in+// the exclude list (exclude taking precedence).+func (f *Filter) ShouldInclude(object string) bool {+ if f == nil {+ return true+ }+ exclude, include := len(f.exclude), len(f.include)+ if exclude == 0 && include == 0 {+ return true+ } else if exclude > 0 && include == 0 {+ return f.shouldIncludeFromExclude(object)+ } else if exclude == 0 && include > 0 {+ return f.shouldIncludeFromInclude(object)+ } else {+ if ok, err := f.shouldIncludeFromBoth(object); err == nil {+ return ok+ }+ // Ambiguous case.+ return false+ }+}++// shouldIncludeFromExclude checks for explicitly excluded paths. This should+// only be called when the include list is empty.+func (f *Filter) shouldIncludeFromExclude(object string) bool {+ for _, glob := range f.exclude {+ if glob.Match(object) {+ return false+ }+ }+ return true+}++// shouldIncludeFromInclude checks for explicitly included paths. This should+// only be called when the exclude list is empty.+func (f *Filter) shouldIncludeFromInclude(object string) bool {+ for _, glob := range f.include {+ if glob.Match(object) {+ return true+ }+ }+ return false+}++// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion+// takes precedence. If neither list contains the object, true is returned.+func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {+ // Exclude takes precedence. If we find the object in the exclude list,+ // we should not match.+ for _, glob := range f.exclude {+ if glob.Match(object) {+ return false, nil+ }+ }+ // If we find the object in the include list, we should match.+ for _, glob := range f.include {+ if glob.Match(object) {+ return true, nil+ }+ }+ // If we find it in neither, return an error to let the caller decide.+ return false, fmt.Errorf("ambiguous match")+}diff --git a/pkg/common/glob/glob_test.go b/pkg/common/glob/glob_test.gonew file mode 100644index 000000000..32a445177--- /dev/null+++ b/pkg/common/glob/glob_test.go@@ -0,0 +1,138 @@+package common++import (+ "testing"++ "github.com/stretchr/testify/assert"+ "pgregory.net/rapid"+)++type globTest struct {+ input string+ shouldInclude bool+}++func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {+ for _, tt := range tests {+ t.Run(tt.input, func(t *testing.T) {+ // Invert because mentally it's easier to say whether an+ // input should be included.+ assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))+ })+ }+}++func TestGlobFilterExclude(t *testing.T) {+ filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))+ assert.NoError(t, err)++ testGlobs(t, filter,+ globTest{"foo", false},+ globTest{"bar", false},+ globTest{"bara", false},+ globTest{"barb", false},+ globTest{"barbosa", false},+ globTest{"foobar", true},+ globTest{"food", true},+ globTest{"anything else", true},+ )+}++func TestGlobFilterInclude(t *testing.T) {+ filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))+ assert.NoError(t, err)++ testGlobs(t, filter,+ globTest{"foo", true},+ globTest{"bar", true},+ globTest{"bara", true},+ globTest{"barb", true},+ globTest{"barbosa", true},+ globTest{"foobar", false},+ globTest{"food", false},+ globTest{"anything else", false},+ )+}++func TestGlobFilterEmpty(t *testing.T) {+ filter, err := NewGlobFilter()+ assert.NoError(t, err)++ testGlobs(t, filter,+ globTest{"foo", true},+ globTest{"bar", true},+ globTest{"bara", true},+ globTest{"barb", true},+ globTest{"barbosa", true},+ globTest{"foobar", true},+ globTest{"food", true},+ globTest{"anything else", true},+ )+}++func TestGlobFilterExcludeInclude(t *testing.T) {+ filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))+ assert.NoError(t, err)++ testGlobs(t, filter,+ globTest{"/foo/a", true},+ globTest{"/foo/b", true},+ globTest{"/foo/c/d/e", true},+ globTest{"/foo/bar/a", false},+ globTest{"/foo/bar/b", false},+ globTest{"/foo/bar/c/d/e", false},+ globTest{"/any/other/path", false},+ )+}++func TestGlobFilterExcludePrecedence(t *testing.T) {+ filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))+ assert.NoError(t, err)++ testGlobs(t, filter,+ globTest{"foo", false},+ globTest{"foobar", true},+ )+}++func TestGlobErrorContainsGlob(t *testing.T) {+ invalidGlob := "[this is invalid because it doesn't close the capture group"+ _, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))+ assert.Error(t, err)+ assert.Contains(t, err.Error(), invalidGlob)+}++// The filters in this test should be mutually exclusive because one includes+// and the other excludes the same glob.+func TestGlobInverse(t *testing.T) {+ for _, glob := range []string{+ "a",+ "a*",+ "a**",+ "*a",+ "**a",+ "*",+ } {+ include, err := NewGlobFilter(WithIncludeGlobs(glob))+ assert.NoError(t, err)+ exclude, err := NewGlobFilter(WithExcludeGlobs(glob))+ assert.NoError(t, err)+ rapid.Check(t, func(t *rapid.T) {+ input := rapid.String().Draw(t, "input")+ a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)+ if a == b {+ t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)+ }+ })+ }+}++func TestGlobDefaultFilters(t *testing.T) {+ for _, filter := range []*Filter{nil, {}} {+ rapid.Check(t, func(t *rapid.T) {+ if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {+ t.Fatalf("filter %#v did not include input", filter)+ }+ })+ }+}