Skip to content

Find, verify, and analyze leaked credentials

Implement SourceUnitUnmarshaller for all sources (#1416)

Miccahf3152b68850f
diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go
index f6d95e50c..ebf7b6cf2 100644
--- a/pkg/sources/circleci/circleci.go
+++ b/pkg/sources/circleci/circleci.go
@@ -31,10 +31,12 @@ type Source struct {
jobPool *errgroup.Group
sources.Progress
client *http.Client
+ sources.CommonSourceUnitUnmarshaller
}
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
diff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.go
index 17534da8c..102fd55f1 100644
--- a/pkg/sources/docker/docker.go
+++ b/pkg/sources/docker/docker.go
@@ -30,12 +30,14 @@ type Source struct {
verify bool
conn sourcespb.Docker
sources.Progress
+ sources.CommonSourceUnitUnmarshaller
}
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 58e756192..eb6dfcf33 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -39,10 +39,12 @@ type Source struct {
log logr.Logger
filter *common.Filter
sources.Progress
+ sources.CommonSourceUnitUnmarshaller
}
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index 9f6f0103c..f2a0bedbf 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -30,8 +30,9 @@ import (
const defaultCachePersistIncrement = 2500
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -69,6 +70,7 @@ type Source struct {
mu sync.Mutex
sources.Progress // progress is not thread safe
+ sources.CommonSourceUnitUnmarshaller
}
// persistableCache is a wrapper around cache.Cache that allows
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 14ddc1ea9..e269c5309 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -67,8 +67,9 @@ func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName s
}
}
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -875,3 +876,7 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
return nil
}
+
+func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
+ return UnmarshalUnit(data)
+}
diff --git a/pkg/sources/git/unit.go b/pkg/sources/git/unit.go
new file mode 100644
index 000000000..9457e23a6
--- /dev/null
+++ b/pkg/sources/git/unit.go
@@ -0,0 +1,40 @@
+package git
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+ UnitRepo string = "repo"
+ UnitDir string = "dir"
+)
+
+// Ensure SourceUnit implements the interface at compile time.
+var _ sources.SourceUnit = SourceUnit{}
+
+// A git source unit can be two kinds of units: either a local directory path
+// or a remote repository.
+type SourceUnit struct {
+ Kind string `json:"kind"`
+ ID string `json:"id"`
+}
+
+// Implement sources.SourceUnit interface.
+func (u SourceUnit) SourceUnitID() string {
+ return u.ID
+}
+
+// Helper function to unmarshal raw bytes into our SourceUnit struct.
+func UnmarshalUnit(data []byte) (sources.SourceUnit, error) {
+ var unit SourceUnit
+ if err := json.Unmarshal(data, &unit); err != nil {
+ return nil, err
+ }
+ if unit.ID == "" || (unit.Kind != UnitRepo && unit.Kind != UnitDir) {
+ return nil, fmt.Errorf("not a git.SourceUnit")
+ }
+ return unit, nil
+}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 9f413b3c8..365c0ead3 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -69,14 +69,17 @@ type Source struct {
mu sync.Mutex
publicMap map[string]source_metadatapb.Visibility
sources.Progress
+ sources.CommonSourceUnitUnmarshaller
}
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
s.scanOptions = scanOptions
}
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+
var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
// Type returns the type of source.
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 7fb96fdea..a32495925 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -47,10 +47,12 @@ type Source struct {
resumeInfoMutex sync.Mutex
sources.Progress
jobPool *errgroup.Group
+ sources.CommonSourceUnitUnmarshaller
}
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
index 4fef9be10..5481201ca 100644
--- a/pkg/sources/s3/s3.go
+++ b/pkg/sources/s3/s3.go
@@ -46,10 +46,12 @@ type Source struct {
conn *sourcespb.S3
jobPool *errgroup.Group
maxObjectSize int64
+ sources.CommonSourceUnitUnmarshaller
}
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source
func (s *Source) Type() sourcespb.SourceType {
diff --git a/pkg/sources/source_unit.go b/pkg/sources/source_unit.go
index 0c05858a2..8eac88430 100644
--- a/pkg/sources/source_unit.go
+++ b/pkg/sources/source_unit.go
@@ -1,12 +1,37 @@
package sources
+import (
+ "encoding/json"
+ "fmt"
+)
+
+// Ensure CommonSourceUnit implements SourceUnit at compile time.
+var _ SourceUnit = CommonSourceUnit{}
+
// CommonSourceUnit is a common implementation of SourceUnit that Sources can
// use instead of implementing their own types.
type CommonSourceUnit struct {
ID string `json:"source_unit_id"`
}
-// Implement the SourceUnit interface.
+// SourceUnitID implements the SourceUnit interface.
func (c CommonSourceUnit) SourceUnitID() string {
return c.ID
}
+
+// CommonSourceUnitUnmarshaller is an implementation of SourceUnitUnmarshaller
+// for the CommonSourceUnit. A source can embed this struct to gain the
+// functionality of converting []byte to a CommonSourceUnit.
+type CommonSourceUnitUnmarshaller struct{}
+
+// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface.
+func (c CommonSourceUnitUnmarshaller) UnmarshalSourceUnit(data []byte) (SourceUnit, error) {
+ var unit CommonSourceUnit
+ if err := json.Unmarshal(data, &unit); err != nil {
+ return nil, err
+ }
+ if unit.ID == "" {
+ return nil, fmt.Errorf("not a CommonSourceUnit")
+ }
+ return unit, nil
+}