Skip to content

Commit 36f7be0

Browse files
lonmarsDevamanfcpkashifkhan0771shahzadhaider1
authored
added flyio detector (#2381)
* added flyio detector * Add Fly.io Detector with Integration Tests - Adjusted regex patterns for token matching - Introduced a new Fly.io secret detector with functionality to identify and verify Fly.io tokens. - Implemented integration tests to validate the detection and verification logic, including handling of false positives. - Updated existing Fly.io test cases to enhance coverage and accuracy. - Added logic to bypass common false positive checks for valid tokens. * incorporated feedback on PR * Remove "flyio" keyword from Fly.io detector keywords for improved accuracy in secret detection. * use existing false positive filtering flow while excluding AAAAAA which seems to be present in valid tokens --------- Co-authored-by: Amaan Ullah <[email protected]> Co-authored-by: Kashif Khan <[email protected]> Co-authored-by: Shahzad Haider <[email protected]>
1 parent a137f1e commit 36f7be0

File tree

4 files changed

+438
-0
lines changed

4 files changed

+438
-0
lines changed

‎pkg/detectors/flyio/flyio.go

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package flyio
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"strings"
9+
10+
regexp "github.com/wasilibs/go-re2"
11+
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
15+
)
16+
17+
type Scanner struct {
18+
client *http.Client
19+
}
20+
21+
// Ensure the Scanner satisfies the interface at compile time.
22+
var _ detectors.Detector = (*Scanner)(nil)
23+
var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
24+
25+
var (
26+
defaultClient = common.SaneHttpClient()
27+
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
28+
keyPat = regexp.MustCompile(`\b(FlyV1 fm\d+_[A-Za-z0-9+\/=,_-]{500,700})\b`)
29+
)
30+
31+
// Keywords are used for efficiently pre-filtering chunks.
32+
// Use identifiers in the secret preferably, or the provider name.
33+
func (s Scanner) Keywords() []string {
34+
return []string{"FlyV1"}
35+
}
36+
37+
// FromData will find and optionally verify Flyio secrets in a given set of bytes.
38+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
39+
dataStr := string(data)
40+
41+
uniqueMatches := make(map[string]struct{})
42+
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
43+
uniqueMatches[match[1]] = struct{}{}
44+
}
45+
46+
for match := range uniqueMatches {
47+
s1 := detectors.Result{
48+
DetectorType: detectorspb.DetectorType_FlyIO,
49+
Raw: []byte(match),
50+
}
51+
52+
if verify {
53+
client := s.client
54+
if client == nil {
55+
client = defaultClient
56+
}
57+
58+
isVerified, verificationErr := verifyMatch(ctx, client, match)
59+
60+
s1.Verified = isVerified
61+
if verificationErr != nil {
62+
s1.SetVerificationError(verificationErr, match)
63+
}
64+
}
65+
66+
results = append(results, s1)
67+
}
68+
69+
return
70+
}
71+
72+
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
73+
// Not setting org_slug intentionally, as it's not required for the token to be valid.
74+
// Initially, an organization named "personal" is created by FlyIO when the user signs up for an account. We cannot rely on this as it can be deleted.
75+
// 403 is returned if incorrect org_slug is sent.
76+
// 401 is returned if the token is invalid.
77+
// 400 is returned if the token is valid but no org_slug is sent.
78+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.machines.dev/v1/apps?org_slug=", http.NoBody)
79+
if err != nil {
80+
return false, nil
81+
}
82+
req.Header.Add("accept", "application/json")
83+
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
84+
85+
res, err := client.Do(req)
86+
if err != nil {
87+
return false, err
88+
}
89+
defer func() {
90+
_, _ = io.Copy(io.Discard, res.Body)
91+
_ = res.Body.Close()
92+
}()
93+
94+
switch res.StatusCode {
95+
case http.StatusBadRequest:
96+
// Not setting org_slug returns a 400 error, which is expected.
97+
return true, nil
98+
case http.StatusUnauthorized:
99+
// The secret is determinately not verified (nothing to do)
100+
return false, nil
101+
default:
102+
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
103+
return false, err
104+
}
105+
}
106+
107+
func (s Scanner) Type() detectorspb.DetectorType {
108+
return detectorspb.DetectorType_FlyIO
109+
}
110+
111+
func (s Scanner) Description() string {
112+
return "Fly.io is a platform for running applications globally. Fly.io tokens can be used to access the Fly.io API and manage applications."
113+
}
114+
115+
func (s Scanner) IsFalsePositive(result detectors.Result) (bool, string) {
116+
// ignore AAAAAA for Flyio detector
117+
if strings.Contains(string(result.Raw), "AAAAAA") {
118+
return false, ""
119+
}
120+
121+
// For non-matching patterns, fall back to default false positive logic
122+
return detectors.IsKnownFalsePositive(string(result.Raw), detectors.DefaultFalsePositives, true)
123+
}

‎pkg/detectors/flyio/flyio_integration_test.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package flyio
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
func TestFlyio_FromChunk(t *testing.T) {
21+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
22+
defer cancel()
23+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
24+
if err != nil {
25+
t.Fatalf("could not get test secrets from GCP: %s", err)
26+
}
27+
secret := testSecrets.MustGetField("FLYIO")
28+
inactiveSecret := testSecrets.MustGetField("FLYIO_INACTIVE")
29+
30+
type args struct {
31+
ctx context.Context
32+
data []byte
33+
verify bool
34+
}
35+
tests := []struct {
36+
name string
37+
s Scanner
38+
args args
39+
want []detectors.Result
40+
wantErr bool
41+
wantVerificationErr bool
42+
}{
43+
{
44+
name: "found, verified",
45+
s: Scanner{},
46+
args: args{
47+
ctx: context.Background(),
48+
data: []byte(fmt.Sprintf("You can find a flyio secret %s within", secret)),
49+
verify: true,
50+
},
51+
want: []detectors.Result{
52+
{
53+
DetectorType: detectorspb.DetectorType_FlyIO,
54+
Verified: true,
55+
},
56+
},
57+
wantErr: false,
58+
wantVerificationErr: false,
59+
},
60+
{
61+
name: "found, unverified",
62+
s: Scanner{},
63+
args: args{
64+
ctx: context.Background(),
65+
data: []byte(fmt.Sprintf("You can find a flyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
66+
verify: true,
67+
},
68+
want: []detectors.Result{
69+
{
70+
DetectorType: detectorspb.DetectorType_FlyIO,
71+
Verified: false,
72+
},
73+
},
74+
wantErr: false,
75+
wantVerificationErr: false,
76+
},
77+
{
78+
name: "not found",
79+
s: Scanner{},
80+
args: args{
81+
ctx: context.Background(),
82+
data: []byte("You cannot find the secret within"),
83+
verify: true,
84+
},
85+
want: nil,
86+
wantErr: false,
87+
wantVerificationErr: false,
88+
},
89+
{
90+
name: "found, would be verified if not for timeout",
91+
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
92+
args: args{
93+
ctx: context.Background(),
94+
data: []byte(fmt.Sprintf("You can find a flyio secret %s within", secret)),
95+
verify: true,
96+
},
97+
want: []detectors.Result{
98+
{
99+
DetectorType: detectorspb.DetectorType_FlyIO,
100+
Verified: false,
101+
},
102+
},
103+
wantErr: false,
104+
wantVerificationErr: true,
105+
},
106+
{
107+
name: "found, verified but unexpected api surface",
108+
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
109+
args: args{
110+
ctx: context.Background(),
111+
data: []byte(fmt.Sprintf("You can find a flyio secret %s within", secret)),
112+
verify: true,
113+
},
114+
want: func() []detectors.Result {
115+
r := detectors.Result{
116+
DetectorType: detectorspb.DetectorType_FlyIO,
117+
Verified: false,
118+
}
119+
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"))
120+
return []detectors.Result{r}
121+
}(),
122+
wantErr: false,
123+
wantVerificationErr: true,
124+
},
125+
}
126+
for _, tt := range tests {
127+
t.Run(tt.name, func(t *testing.T) {
128+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
129+
if (err != nil) != tt.wantErr {
130+
t.Errorf("Flyio.FromData() error = %v, wantErr %v", err, tt.wantErr)
131+
return
132+
}
133+
for i := range got {
134+
if len(got[i].Raw) == 0 {
135+
t.Fatalf("no raw secret present: \n %+v", got[i])
136+
}
137+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
138+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
139+
}
140+
}
141+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
142+
ignoreUnexported := cmpopts.IgnoreUnexported(detectors.Result{})
143+
if diff := cmp.Diff(got, tt.want, ignoreOpts, ignoreUnexported); diff != "" {
144+
t.Errorf("Flyio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
145+
}
146+
})
147+
}
148+
}

0 commit comments

Comments
 (0)