Skip to content

Find, verify, and analyze leaked credentials

Add jobID to chunk. (#1721)

ahrav2b1b1b5ad06e
diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go
index 1fb94c179..2ae7e6b3b 100644
--- a/pkg/sources/circleci/circleci.go
+++ b/pkg/sources/circleci/circleci.go
@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
diff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.go
index 71b57e6e2..7d3475174 100644
--- a/pkg/sources/docker/docker.go
+++ b/pkg/sources/docker/docker.go
@@ -174,6 +174,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 89d473486..49f504006 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index f2a0bedbf..dfb0a40f3 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
+ JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 9bed59ca6..76301cbae 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
+ JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 302269083..3b5d1f016 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -1225,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1258,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
+ JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1289,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
+ JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
index b05832ed1..44d62572e 100644
--- a/pkg/sources/s3/s3.go
+++ b/pkg/sources/s3/s3.go
@@ -376,6 +376,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
+ JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.go
index b2ba64fa1..d9be9d117 100644
--- a/pkg/sources/syslog/syslog.go
+++ b/pkg/sources/syslog/syslog.go
@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
+ JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
+ JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,