Skip to content

Commit 836ed28

Browse files
kashifkhan0771kashifkhan0771
authored
Fixed gitlab detector (#4371)
* Fixed gitlab detector * moved validations outside endpoints loop * resolved comments * resolved comments
1 parent c9cacfd commit 836ed28

File tree

6 files changed

+85
-79
lines changed

6 files changed

+85
-79
lines changed

‎pkg/detectors/gitlab/v1/gitlab.go

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -67,29 +67,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
6767

6868
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
6969
for _, match := range matches {
70+
resMatch := strings.TrimSpace(match[1])
71+
7072
// ignore v2 detectors which have a prefix of `glpat-`
7173
if strings.Contains(match[0], "glpat-") {
7274
continue
7375
}
74-
resMatch := strings.TrimSpace(match[1])
7576

7677
// to avoid false positives
7778
if detectors.StringShannonEntropy(resMatch) < 3.6 {
7879
continue
7980
}
8081

81-
s1 := detectors.Result{
82-
DetectorType: detectorspb.DetectorType_Gitlab,
83-
Raw: []byte(resMatch),
84-
ExtraData: map[string]string{},
85-
}
86-
s1.ExtraData = map[string]string{
87-
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
88-
"version": fmt.Sprintf("%d", s.Version()),
89-
}
82+
for _, endpoint := range s.Endpoints() {
83+
s1 := detectors.Result{
84+
DetectorType: detectorspb.DetectorType_Gitlab,
85+
Raw: []byte(resMatch),
86+
RawV2: []byte(resMatch + endpoint),
87+
ExtraData: map[string]string{
88+
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
89+
"version": fmt.Sprintf("%d", s.Version()),
90+
},
91+
}
9092

91-
if verify {
92-
for _, endpoint := range s.Endpoints() {
93+
if verify {
9394
isVerified, extraData, verificationErr := VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
9495
s1.Verified = isVerified
9596
maps.Copy(s1.ExtraData, extraData)
@@ -102,11 +103,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
102103
"key": resMatch,
103104
"host": endpoint,
104105
}
106+
107+
// if secret is verified with one endpoint, break the loop to continue to next secret
108+
results = append(results, s1)
109+
break
105110
}
106111
}
107-
}
108112

109-
results = append(results, s1)
113+
results = append(results, s1)
114+
}
110115
}
111116

112117
return results, nil

‎pkg/detectors/gitlab/v1/gitlab_integration_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ func TestGitlab_FromChunk(t *testing.T) {
175175
}
176176
for _, tt := range tests {
177177
t.Run(tt.name, func(t *testing.T) {
178+
tt.s.SetCloudEndpoint("https://gitlab.com")
179+
tt.s.UseCloudEndpoint(true)
178180
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
179181
if (err != nil) != tt.wantErr {
180182
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -189,7 +191,7 @@ func TestGitlab_FromChunk(t *testing.T) {
189191
}
190192
got[i].AnalysisInfo = nil
191193
}
192-
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
194+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
193195
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
194196
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
195197
}
@@ -287,7 +289,7 @@ func TestGitlab_FromChunk_WithV2Secrets(t *testing.T) {
287289
}
288290
got[i].AnalysisInfo = nil
289291
}
290-
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
292+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
291293
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
292294
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
293295
}

‎pkg/detectors/gitlab/v1/gitlab_v1_test.go

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,10 @@ import (
1010
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
1111
)
1212

13-
var (
14-
validPattern = `[{
15-
"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
16-
"name": "Gitlab",
17-
"type": "Detector",
18-
"api": true,
19-
"authentication_type": "",
20-
"verification_url": "https://api.example.com/example",
21-
"test_secrets": {
22-
"gitlab_secret": "oXCt4JT2wf1_WlZl2OVG"
23-
},
24-
"docs":"https://docs.gitlab.com/test/api/example.json#get-drone-test-example-settings", // this matches the pattern but fail in entropy check
25-
"expected_response": "200",
26-
"method": "GET",
27-
"deprecated": false
28-
}]`
29-
secret = "oXCt4JT2wf1_WlZl2OVG"
30-
validPattern2 = "GITLAB_TOKEN=ABc123456789dEFghIJK"
31-
secret2 = "ABc123456789dEFghIJK"
32-
)
33-
3413
func TestGitLab_Pattern(t *testing.T) {
3514
d := Scanner{}
15+
d.SetCloudEndpoint("https://gitlab.com")
16+
d.UseCloudEndpoint(true)
3617
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
3718

3819
tests := []struct {
@@ -41,14 +22,28 @@ func TestGitLab_Pattern(t *testing.T) {
4122
want []string
4223
}{
4324
{
44-
name: "valid pattern",
45-
input: validPattern,
46-
want: []string{secret},
25+
name: "valid pattern",
26+
input: `[{
27+
"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
28+
"name": "Gitlab",
29+
"type": "Detector",
30+
"api": true,
31+
"authentication_type": "",
32+
"verification_url": "https://api.example.com/example",
33+
"test_secrets": {
34+
"gitlab_secret": "oXCt4JT2wf1_WlZl2OVG"
35+
},
36+
"docs":"https://docs.gitlab.com/test/api/example.json#get-drone-test-example-settings", // this matches the pattern but fail in entropy check
37+
"expected_response": "200",
38+
"method": "GET",
39+
"deprecated": false
40+
}]`,
41+
want: []string{"oXCt4JT2wf1_WlZl2OVGhttps://gitlab.com"},
4742
},
4843
{
4944
name: "valid pattern (with = before secret)",
50-
input: validPattern2,
51-
want: []string{secret2},
45+
input: "GITLAB_TOKEN=ABc123456789dEFghIJK",
46+
want: []string{"ABc123456789dEFghIJKhttps://gitlab.com"},
5247
},
5348
}
5449

‎pkg/detectors/gitlab/v2/gitlab_integration_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
8787
}
8888
for _, tt := range tests {
8989
t.Run(tt.name, func(t *testing.T) {
90+
tt.s.SetCloudEndpoint("https://gitlab.com")
91+
tt.s.UseCloudEndpoint(true)
9092
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
9193
if (err != nil) != tt.wantErr {
9294
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -101,7 +103,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
101103
}
102104
got[i].AnalysisInfo = nil
103105
}
104-
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
106+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
105107
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
106108
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
107109
}
@@ -280,7 +282,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {
280282
}
281283
got[i].AnalysisInfo = nil
282284
}
283-
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
285+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
284286
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
285287
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
286288
}

‎pkg/detectors/gitlab/v2/gitlab_v2.go

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -59,20 +59,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
5959

6060
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
6161
for _, match := range matches {
62-
6362
resMatch := strings.TrimSpace(match[1])
64-
s1 := detectors.Result{
65-
DetectorType: detectorspb.DetectorType_Gitlab,
66-
Raw: []byte(resMatch),
67-
ExtraData: map[string]string{},
68-
}
69-
s1.ExtraData = map[string]string{
70-
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
71-
"version": fmt.Sprintf("%d", s.Version()),
72-
}
7363

74-
if verify {
75-
for _, endpoint := range s.Endpoints() {
64+
for _, endpoint := range s.Endpoints() {
65+
s1 := detectors.Result{
66+
DetectorType: detectorspb.DetectorType_Gitlab,
67+
Raw: []byte(resMatch),
68+
RawV2: []byte(resMatch + endpoint),
69+
ExtraData: map[string]string{
70+
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
71+
"version": fmt.Sprintf("%d", s.Version()),
72+
},
73+
}
74+
75+
if verify {
76+
7677
isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
7778
s1.Verified = isVerified
7879
maps.Copy(s1.ExtraData, extraData)
@@ -85,11 +86,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
8586
"key": resMatch,
8687
"host": endpoint,
8788
}
89+
90+
// if secret is verified with one endpoint, break the loop to continue to next secret
91+
results = append(results, s1)
92+
break
8893
}
8994
}
90-
}
9195

92-
results = append(results, s1)
96+
results = append(results, s1)
97+
}
9398
}
9499

95100
return results, nil

‎pkg/detectors/gitlab/v2/gitlab_v2_test.go

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,10 @@ import (
1010
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
1111
)
1212

13-
var (
14-
validPattern = `[{
15-
"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
16-
"name": "Gitlab",
17-
"type": "Detector",
18-
"api": true,
19-
"authentication_type": "",
20-
"verification_url": "https://api.example.com/example",
21-
"test_secrets": {
22-
"gitlab_secret": "glpat-W6fYSu70dPEo5w_SwbHWgQ"
23-
},
24-
"expected_response": "200",
25-
"method": "GET",
26-
"deprecated": false
27-
}]`
28-
secret = "glpat-W6fYSu70dPEo5w_SwbHWgQ"
29-
)
30-
3113
func TestGitLab_Pattern(t *testing.T) {
3214
d := Scanner{}
15+
d.SetCloudEndpoint("https://gitlab.com")
16+
d.UseCloudEndpoint(true)
3317
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
3418

3519
tests := []struct {
@@ -38,9 +22,22 @@ func TestGitLab_Pattern(t *testing.T) {
3822
want []string
3923
}{
4024
{
41-
name: "valid pattern",
42-
input: validPattern,
43-
want: []string{secret},
25+
name: "valid pattern",
26+
input: `[{
27+
"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
28+
"name": "Gitlab",
29+
"type": "Detector",
30+
"api": true,
31+
"authentication_type": "",
32+
"verification_url": "https://api.example.com/example",
33+
"test_secrets": {
34+
"gitlab_secret": "glpat-W6fYSu70dPEo5w_SwbHWgQ"
35+
},
36+
"expected_response": "200",
37+
"method": "GET",
38+
"deprecated": false
39+
}]`,
40+
want: []string{"glpat-W6fYSu70dPEo5w_SwbHWgQhttps://gitlab.com"},
4441
},
4542
}
4643

0 commit comments

Comments
 (0)