Skip to content

Find, verify, and analyze leaked credentials

Add generic glob filter (#1858)

Miccah23ae970bb001
diff --git a/go.mod b/go.mod
index 64029ba20..a09acc9b5 100644
--- a/go.mod
+++ b/go.mod
@@ -81,6 +81,7 @@ require (
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
+ pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.3.0
)
diff --git a/go.sum b/go.sum
index 7fc55883c..704cc19c5 100644
--- a/go.sum
+++ b/go.sum
@@ -1013,6 +1013,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/pkg/common/glob/glob.go b/pkg/common/glob/glob.go
new file mode 100644
index 000000000..a31bbe6fc
--- /dev/null
+++ b/pkg/common/glob/glob.go
@@ -0,0 +1,120 @@
+package common
+
+import (
+ "fmt"
+
+ "github.com/gobwas/glob"
+)
+
+// Filter is a generic filter for excluding and including globs (limited
+// regular expressions). Exclusion takes precedence if both include and exclude
+// lists are provided.
+type Filter struct {
+ exclude []glob.Glob
+ include []glob.Glob
+}
+
+type globFilterOpt func(*Filter) error
+
+// WithExcludeGlobs adds exclude globs to the filter.
+func WithExcludeGlobs(excludes ...string) globFilterOpt {
+ return func(f *Filter) error {
+ for _, exclude := range excludes {
+ g, err := glob.Compile(exclude)
+ if err != nil {
+ return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
+ }
+ f.exclude = append(f.exclude, g)
+ }
+ return nil
+ }
+}
+
+// WithIncludeGlobs adds include globs to the filter.
+func WithIncludeGlobs(includes ...string) globFilterOpt {
+ return func(f *Filter) error {
+ for _, include := range includes {
+ g, err := glob.Compile(include)
+ if err != nil {
+ return fmt.Errorf("invalid include glob %q: %w", include, err)
+ }
+ f.include = append(f.include, g)
+ }
+ return nil
+ }
+}
+
+// NewGlobFilter creates a new Filter with the provided options.
+func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
+ filter := &Filter{}
+ for _, opt := range opts {
+ if err := opt(filter); err != nil {
+ return nil, err
+ }
+ }
+ return filter, nil
+}
+
+// ShouldInclude returns whether the object is in the include list or not in
+// the exclude list (exclude taking precedence).
+func (f *Filter) ShouldInclude(object string) bool {
+ if f == nil {
+ return true
+ }
+ exclude, include := len(f.exclude), len(f.include)
+ if exclude == 0 && include == 0 {
+ return true
+ } else if exclude > 0 && include == 0 {
+ return f.shouldIncludeFromExclude(object)
+ } else if exclude == 0 && include > 0 {
+ return f.shouldIncludeFromInclude(object)
+ } else {
+ if ok, err := f.shouldIncludeFromBoth(object); err == nil {
+ return ok
+ }
+ // Ambiguous case.
+ return false
+ }
+}
+
+// shouldIncludeFromExclude checks for explicitly excluded paths. This should
+// only be called when the include list is empty.
+func (f *Filter) shouldIncludeFromExclude(object string) bool {
+ for _, glob := range f.exclude {
+ if glob.Match(object) {
+ return false
+ }
+ }
+ return true
+}
+
+// shouldIncludeFromInclude checks for explicitly included paths. This should
+// only be called when the exclude list is empty.
+func (f *Filter) shouldIncludeFromInclude(object string) bool {
+ for _, glob := range f.include {
+ if glob.Match(object) {
+ return true
+ }
+ }
+ return false
+}
+
+// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
+// takes precedence. If neither list contains the object, true is returned.
+func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
+ // Exclude takes precedence. If we find the object in the exclude list,
+ // we should not match.
+ for _, glob := range f.exclude {
+ if glob.Match(object) {
+ return false, nil
+ }
+ }
+ // If we find the object in the include list, we should match.
+ for _, glob := range f.include {
+ if glob.Match(object) {
+ return true, nil
+ }
+ }
+ // If we find it in neither, return an error to let the caller decide.
+ return false, fmt.Errorf("ambiguous match")
+}
diff --git a/pkg/common/glob/glob_test.go b/pkg/common/glob/glob_test.go
new file mode 100644
index 000000000..32a445177
--- /dev/null
+++ b/pkg/common/glob/glob_test.go
@@ -0,0 +1,138 @@
+package common
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "pgregory.net/rapid"
+)
+
+type globTest struct {
+ input string
+ shouldInclude bool
+}
+
+func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
+ for _, tt := range tests {
+ t.Run(tt.input, func(t *testing.T) {
+ // Invert because mentally it's easier to say whether an
+ // input should be included.
+ assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
+ })
+ }
+}
+
+func TestGlobFilterExclude(t *testing.T) {
+ filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
+ assert.NoError(t, err)
+
+ testGlobs(t, filter,
+ globTest{"foo", false},
+ globTest{"bar", false},
+ globTest{"bara", false},
+ globTest{"barb", false},
+ globTest{"barbosa", false},
+ globTest{"foobar", true},
+ globTest{"food", true},
+ globTest{"anything else", true},
+ )
+}
+
+func TestGlobFilterInclude(t *testing.T) {
+ filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
+ assert.NoError(t, err)
+
+ testGlobs(t, filter,
+ globTest{"foo", true},
+ globTest{"bar", true},
+ globTest{"bara", true},
+ globTest{"barb", true},
+ globTest{"barbosa", true},
+ globTest{"foobar", false},
+ globTest{"food", false},
+ globTest{"anything else", false},
+ )
+}
+
+func TestGlobFilterEmpty(t *testing.T) {
+ filter, err := NewGlobFilter()
+ assert.NoError(t, err)
+
+ testGlobs(t, filter,
+ globTest{"foo", true},
+ globTest{"bar", true},
+ globTest{"bara", true},
+ globTest{"barb", true},
+ globTest{"barbosa", true},
+ globTest{"foobar", true},
+ globTest{"food", true},
+ globTest{"anything else", true},
+ )
+}
+
+func TestGlobFilterExcludeInclude(t *testing.T) {
+ filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
+ assert.NoError(t, err)
+
+ testGlobs(t, filter,
+ globTest{"/foo/a", true},
+ globTest{"/foo/b", true},
+ globTest{"/foo/c/d/e", true},
+ globTest{"/foo/bar/a", false},
+ globTest{"/foo/bar/b", false},
+ globTest{"/foo/bar/c/d/e", false},
+ globTest{"/any/other/path", false},
+ )
+}
+
+func TestGlobFilterExcludePrecedence(t *testing.T) {
+ filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
+ assert.NoError(t, err)
+
+ testGlobs(t, filter,
+ globTest{"foo", false},
+ globTest{"foobar", true},
+ )
+}
+
+func TestGlobErrorContainsGlob(t *testing.T) {
+ invalidGlob := "[this is invalid because it doesn't close the capture group"
+ _, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), invalidGlob)
+}
+
+// The filters in this test should be mutually exclusive because one includes
+// and the other excludes the same glob.
+func TestGlobInverse(t *testing.T) {
+ for _, glob := range []string{
+ "a",
+ "a*",
+ "a**",
+ "*a",
+ "**a",
+ "*",
+ } {
+ include, err := NewGlobFilter(WithIncludeGlobs(glob))
+ assert.NoError(t, err)
+ exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
+ assert.NoError(t, err)
+ rapid.Check(t, func(t *rapid.T) {
+ input := rapid.String().Draw(t, "input")
+ a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
+ if a == b {
+ t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
+ }
+ })
+ }
+}
+
+func TestGlobDefaultFilters(t *testing.T) {
+ for _, filter := range []*Filter{nil, {}} {
+ rapid.Check(t, func(t *rapid.T) {
+ if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
+ t.Fatalf("filter %#v did not include input", filter)
+ }
+ })
+ }
+}