Skip to content

Commit 6445fcd

Browse files
caffixcaffix
committed
started to integrate the new asset db cache
1 parent 1e610c5 commit 6445fcd

32 files changed

+431
-878
lines changed

‎engine/plugins/api/binaryedge.go

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,21 @@ import (
2020
dbt "github.com/owasp-amass/asset-db/types"
2121
oam "github.com/owasp-amass/open-asset-model"
2222
"github.com/owasp-amass/open-asset-model/domain"
23-
"github.com/owasp-amass/open-asset-model/source"
2423
"go.uber.org/ratelimit"
2524
)
2625

2726
type binaryEdge struct {
2827
name string
2928
log *slog.Logger
3029
rlimit ratelimit.Limiter
31-
source *source.Source
30+
source *et.Source
3231
}
3332

3433
func NewBinaryEdge() et.Plugin {
3534
return &binaryEdge{
3635
name: "BinaryEdge",
3736
rlimit: ratelimit.New(10, ratelimit.WithoutSlack),
38-
source: &source.Source{
37+
source: &et.Source{
3938
Name: "BinaryEdge",
4039
Confidence: 80,
4140
},
@@ -70,7 +69,7 @@ func (be *binaryEdge) Stop() {
7069
}
7170

7271
func (be *binaryEdge) check(e *et.Event) error {
73-
fqdn, ok := e.Asset.Asset.(*domain.FQDN)
72+
fqdn, ok := e.Entity.Asset.(*domain.FQDN)
7473
if !ok {
7574
return errors.New("failed to extract the FQDN asset")
7675
}
@@ -103,7 +102,7 @@ func (be *binaryEdge) check(e *et.Event) error {
103102
return err
104103
}
105104

106-
var names []*dbt.Asset
105+
var names []*dbt.Entity
107106
if support.AssetMonitoredWithinTTL(e.Session, e.Asset, src, since) {
108107
names = append(names, be.lookup(e, fqdn.Name, src, since)...)
109108
} else {
@@ -117,11 +116,11 @@ func (be *binaryEdge) check(e *et.Event) error {
117116
return nil
118117
}
119118

120-
func (be *binaryEdge) lookup(e *et.Event, name string, src *dbt.Asset, since time.Time) []*dbt.Asset {
119+
func (be *binaryEdge) lookup(e *et.Event, name string, src *dbt.Entity, since time.Time) []*dbt.Entity {
121120
return support.SourceToAssetsWithinTTL(e.Session, name, string(oam.FQDN), src, since)
122121
}
123122

124-
func (be *binaryEdge) query(e *et.Event, name string, src *dbt.Asset, keys []string) []*dbt.Asset {
123+
func (be *binaryEdge) query(e *et.Event, name string, src *et.Source, keys []string) []*dbt.Entity {
125124
subs := stringset.New()
126125
defer subs.Close()
127126

@@ -170,10 +169,10 @@ loop:
170169
return be.store(e, subs.Slice(), src)
171170
}
172171

173-
func (be *binaryEdge) store(e *et.Event, names []string, src *dbt.Asset) []*dbt.Asset {
172+
func (be *binaryEdge) store(e *et.Event, names []string, src *et.Source) []*dbt.Entity {
174173
return support.StoreFQDNsWithSource(e.Session, names, src, be.name, be.name+"-Handler")
175174
}
176175

177-
func (be *binaryEdge) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) {
176+
func (be *binaryEdge) process(e *et.Event, assets []*dbt.Entity, src *et.Source) {
178177
support.ProcessFQDNsWithSource(e, assets, src)
179178
}

‎engine/plugins/api/chaos.go

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,21 @@ import (
1919
dbt "github.com/owasp-amass/asset-db/types"
2020
oam "github.com/owasp-amass/open-asset-model"
2121
"github.com/owasp-amass/open-asset-model/domain"
22-
"github.com/owasp-amass/open-asset-model/source"
2322
"go.uber.org/ratelimit"
2423
)
2524

2625
type chaos struct {
2726
name string
2827
log *slog.Logger
2928
rlimit ratelimit.Limiter
30-
source *source.Source
29+
source *et.Source
3130
}
3231

3332
func NewChaos() et.Plugin {
3433
return &chaos{
3534
name: "Chaos",
3635
rlimit: ratelimit.New(10, ratelimit.WithoutSlack),
37-
source: &source.Source{
36+
source: &et.Source{
3837
Name: "Chaos",
3938
Confidence: 80,
4039
},
@@ -69,7 +68,7 @@ func (c *chaos) Stop() {
6968
}
7069

7170
func (c *chaos) check(e *et.Event) error {
72-
fqdn, ok := e.Asset.Asset.(*domain.FQDN)
71+
fqdn, ok := e.Entity.Asset.(*domain.FQDN)
7372
if !ok {
7473
return errors.New("failed to extract the FQDN asset")
7574
}
@@ -102,7 +101,7 @@ func (c *chaos) check(e *et.Event) error {
102101
return err
103102
}
104103

105-
var names []*dbt.Asset
104+
var names []*dbt.Entity
106105
if support.AssetMonitoredWithinTTL(e.Session, e.Asset, src, since) {
107106
names = append(names, c.lookup(e, fqdn.Name, src, since)...)
108107
} else {
@@ -116,11 +115,11 @@ func (c *chaos) check(e *et.Event) error {
116115
return nil
117116
}
118117

119-
func (c *chaos) lookup(e *et.Event, name string, src *dbt.Asset, since time.Time) []*dbt.Asset {
118+
func (c *chaos) lookup(e *et.Event, name string, src *dbt.Entity, since time.Time) []*dbt.Entity {
120119
return support.SourceToAssetsWithinTTL(e.Session, name, string(oam.FQDN), src, since)
121120
}
122121

123-
func (c *chaos) query(e *et.Event, name string, src *dbt.Asset, keys []string) []*dbt.Asset {
122+
func (c *chaos) query(e *et.Event, name string, src *dbt.Entity, keys []string) []*dbt.Entity {
124123
var names []string
125124

126125
for _, key := range keys {
@@ -152,10 +151,10 @@ func (c *chaos) query(e *et.Event, name string, src *dbt.Asset, keys []string) [
152151
return c.store(e, names, src)
153152
}
154153

155-
func (c *chaos) store(e *et.Event, names []string, src *dbt.Asset) []*dbt.Asset {
154+
func (c *chaos) store(e *et.Event, names []string, src *dbt.Entity) []*dbt.Entity {
156155
return support.StoreFQDNsWithSource(e.Session, names, src, c.name, c.name+"-Handler")
157156
}
158157

159-
func (c *chaos) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) {
158+
func (c *chaos) process(e *et.Event, assets []*dbt.Entity, src *dbt.Entity) {
160159
support.ProcessFQDNsWithSource(e, assets, src)
161160
}

‎engine/plugins/api/crtsh.go

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,21 @@ import (
1919
dbt "github.com/owasp-amass/asset-db/types"
2020
oam "github.com/owasp-amass/open-asset-model"
2121
"github.com/owasp-amass/open-asset-model/domain"
22-
"github.com/owasp-amass/open-asset-model/source"
2322
"go.uber.org/ratelimit"
2423
)
2524

2625
type crtsh struct {
2726
name string
2827
log *slog.Logger
2928
rlimit ratelimit.Limiter
30-
source *source.Source
29+
source *et.Source
3130
}
3231

3332
func NewCrtsh() et.Plugin {
3433
return &crtsh{
3534
name: "crt.sh",
3635
rlimit: ratelimit.New(2, ratelimit.WithoutSlack),
37-
source: &source.Source{
36+
source: &et.Source{
3837
Name: "HackerTarget",
3938
Confidence: 100,
4039
},
@@ -69,7 +68,7 @@ func (c *crtsh) Stop() {
6968
}
7069

7170
func (c *crtsh) check(e *et.Event) error {
72-
fqdn, ok := e.Asset.Asset.(*domain.FQDN)
71+
fqdn, ok := e.Entity.Asset.(*domain.FQDN)
7372
if !ok {
7473
return errors.New("failed to extract the FQDN asset")
7574
}
@@ -90,7 +89,7 @@ func (c *crtsh) check(e *et.Event) error {
9089
return err
9190
}
9291

93-
var names []*dbt.Asset
92+
var names []*dbt.Entity
9493
if support.AssetMonitoredWithinTTL(e.Session, e.Asset, src, since) {
9594
names = append(names, c.lookup(e, fqdn.Name, src, since)...)
9695
} else {
@@ -104,17 +103,17 @@ func (c *crtsh) check(e *et.Event) error {
104103
return nil
105104
}
106105

107-
func (c *crtsh) lookup(e *et.Event, name string, src *dbt.Asset, since time.Time) []*dbt.Asset {
106+
func (c *crtsh) lookup(e *et.Event, name string, src *et.Source, since time.Time) []*dbt.Entity {
108107
return support.SourceToAssetsWithinTTL(e.Session, name, string(oam.FQDN), src, since)
109108
}
110109

111-
func (c *crtsh) query(e *et.Event, name string, src *dbt.Asset) []*dbt.Asset {
110+
func (c *crtsh) query(e *et.Event, name string, src *et.Source) []*dbt.Entity {
112111
c.rlimit.Take()
113112
resp, err := http.RequestWebPage(context.TODO(), &http.Request{
114113
URL: "https://crt.sh/?CN=" + name + "&output=json&exclude=expired",
115114
})
116115
if err != nil {
117-
return []*dbt.Asset{}
116+
return nil
118117
}
119118

120119
var result struct {
@@ -123,7 +122,7 @@ func (c *crtsh) query(e *et.Event, name string, src *dbt.Asset) []*dbt.Asset {
123122
} `json:"certs"`
124123
}
125124
if err := json.Unmarshal([]byte("{\"certs\":"+resp.Body+"}"), &result); err != nil {
126-
return []*dbt.Asset{}
125+
return nil
127126
}
128127

129128
var names []string
@@ -140,10 +139,10 @@ func (c *crtsh) query(e *et.Event, name string, src *dbt.Asset) []*dbt.Asset {
140139
return c.store(e, names, src)
141140
}
142141

143-
func (c *crtsh) store(e *et.Event, names []string, src *dbt.Asset) []*dbt.Asset {
142+
func (c *crtsh) store(e *et.Event, names []string, src *et.Source) []*dbt.Entity {
144143
return support.StoreFQDNsWithSource(e.Session, names, src, c.name, c.name+"-Handler")
145144
}
146145

147-
func (c *crtsh) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) {
146+
func (c *crtsh) process(e *et.Event, assets []*dbt.Entity, src *et.Source) {
148147
support.ProcessFQDNsWithSource(e, assets, src)
149148
}

‎engine/plugins/api/dnsrepo.go

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,21 @@ import (
2020
dbt "github.com/owasp-amass/asset-db/types"
2121
oam "github.com/owasp-amass/open-asset-model"
2222
"github.com/owasp-amass/open-asset-model/domain"
23-
"github.com/owasp-amass/open-asset-model/source"
2423
"go.uber.org/ratelimit"
2524
)
2625

2726
type dnsrepo struct {
2827
name string
2928
log *slog.Logger
3029
rlimit ratelimit.Limiter
31-
source *source.Source
30+
source *et.Source
3231
}
3332

3433
func NewDNSRepo() et.Plugin {
3534
return &dnsrepo{
3635
name: "DNSRepo",
3736
rlimit: ratelimit.New(10, ratelimit.WithoutSlack),
38-
source: &source.Source{
37+
source: &et.Source{
3938
Name: "DNSRepo",
4039
Confidence: 80,
4140
},
@@ -70,7 +69,7 @@ func (d *dnsrepo) Stop() {
7069
}
7170

7271
func (d *dnsrepo) check(e *et.Event) error {
73-
fqdn, ok := e.Asset.Asset.(*domain.FQDN)
72+
fqdn, ok := e.Entity.Asset.(*domain.FQDN)
7473
if !ok {
7574
return errors.New("failed to extract the FQDN asset")
7675
}
@@ -101,7 +100,7 @@ func (d *dnsrepo) check(e *et.Event) error {
101100
return err
102101
}
103102

104-
var names []*dbt.Asset
103+
var names []*dbt.Entity
105104
if support.AssetMonitoredWithinTTL(e.Session, e.Asset, src, since) {
106105
names = append(names, d.lookup(e, fqdn.Name, src, since)...)
107106
} else {
@@ -115,11 +114,11 @@ func (d *dnsrepo) check(e *et.Event) error {
115114
return nil
116115
}
117116

118-
func (d *dnsrepo) lookup(e *et.Event, name string, src *dbt.Asset, since time.Time) []*dbt.Asset {
117+
func (d *dnsrepo) lookup(e *et.Event, name string, src *et.Source, since time.Time) []*dbt.Entity {
119118
return support.SourceToAssetsWithinTTL(e.Session, name, string(oam.FQDN), src, since)
120119
}
121120

122-
func (d *dnsrepo) query(e *et.Event, name string, src *dbt.Asset, keys []string) []*dbt.Asset {
121+
func (d *dnsrepo) query(e *et.Event, name string, src *et.Source, keys []string) []*dbt.Entity {
123122
var names []string
124123

125124
for _, key := range keys {
@@ -202,10 +201,10 @@ func (d *dnsrepo) parseJSON(e *et.Event, body string) []string {
202201
return set.Slice()
203202
}
204203

205-
func (d *dnsrepo) store(e *et.Event, names []string, src *dbt.Asset) []*dbt.Asset {
204+
func (d *dnsrepo) store(e *et.Event, names []string, src *et.Source) []*dbt.Entity {
206205
return support.StoreFQDNsWithSource(e.Session, names, src, d.name, d.name+"-Handler")
207206
}
208207

209-
func (d *dnsrepo) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) {
208+
func (d *dnsrepo) process(e *et.Event, assets []*dbt.Entity, src *et.Source) {
210209
support.ProcessFQDNsWithSource(e, assets, src)
211210
}

‎engine/plugins/api/grepapp.go

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,22 +23,21 @@ import (
2323
dbt "github.com/owasp-amass/asset-db/types"
2424
oam "github.com/owasp-amass/open-asset-model"
2525
"github.com/owasp-amass/open-asset-model/domain"
26-
"github.com/owasp-amass/open-asset-model/source"
2726
"go.uber.org/ratelimit"
2827
)
2928

3029
type grepApp struct {
3130
name string
3231
log *slog.Logger
3332
rlimit ratelimit.Limiter
34-
source *source.Source
33+
source *et.Source
3534
}
3635

3736
func NewGrepApp() et.Plugin {
3837
return &grepApp{
3938
name: "Grep.App",
4039
rlimit: ratelimit.New(2, ratelimit.WithoutSlack),
41-
source: &source.Source{
40+
source: &et.Source{
4241
Name: "Grep.App",
4342
Confidence: 50,
4443
},
@@ -95,7 +94,7 @@ func (g *grepApp) check(e *et.Event) error {
9594
return err
9695
}
9796

98-
var names []*dbt.Asset
97+
var names []*dbt.Entity
9998
if support.AssetMonitoredWithinTTL(e.Session, e.Asset, src, since) {
10099
names = append(names, g.lookup(e, fqdn.Name, src, since)...)
101100
} else {
@@ -108,11 +107,11 @@ func (g *grepApp) check(e *et.Event) error {
108107
}
109108
return nil
110109
}
111-
func (g *grepApp) lookup(e *et.Event, name string, src *dbt.Asset, since time.Time) []*dbt.Asset {
110+
func (g *grepApp) lookup(e *et.Event, name string, src *et.Source, since time.Time) []*dbt.Entity {
112111
return support.SourceToAssetsWithinTTL(e.Session, name, string(oam.EmailAddress), src, since)
113112
}
114113

115-
func (g *grepApp) query(e *et.Event, name string, src *dbt.Asset) []*dbt.Asset {
114+
func (g *grepApp) query(e *et.Event, name string, src *et.Source) []*dbt.Entity {
116115
newdlt := strings.ReplaceAll(name, ".", `\.`)
117116
escapedQuery := url.QueryEscape("([a-zA-Z0-9._-]+)@" + newdlt)
118117
re := regexp.MustCompile(`([a-zA-Z0-9._-]+)@` + newdlt)
@@ -157,10 +156,10 @@ func (g *grepApp) query(e *et.Event, name string, src *dbt.Asset) []*dbt.Asset {
157156
return g.store(e, emails.Slice(), src)
158157
}
159158

160-
func (g *grepApp) store(e *et.Event, emails []string, src *dbt.Asset) []*dbt.Asset {
159+
func (g *grepApp) store(e *et.Event, emails []string, src *et.Source) []*dbt.Entity {
161160
return support.StoreEmailsWithSource(e.Session, emails, src, g.name, g.name+"-Handler")
162161
}
163162

164-
func (g *grepApp) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) {
163+
func (g *grepApp) process(e *et.Event, assets []*dbt.Entity, src *et.Source) {
165164
support.ProcessEmailsWithSource(e, assets, src)
166165
}

0 commit comments

Comments
 (0)