Refactored with cloudflare v4 library and also added some features and some code cleanup

README.md

@@ -29,6 +29,7 @@ Required authentication scopes:
   Workers included in authentication scope)
 - `Firewall Services:Read` is required to fetch zone rule name for `cloudflare_zone_firewall_events_count` metric
 - `Account. Account Rulesets:Read` is required to fetch account rule name for `cloudflare_zone_firewall_events_count` metric
+- `Account:Load Balancing: Monitors and Pools:Read` is required to fetch pools origin health status `cloudflare_pool_origin_health_status` metric
 
 To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`)
 
@@ -52,9 +53,9 @@ The exporter can be configured using env variables or command flags.
 | `LISTEN` |  listen on addr:port (default `:8080`), omit addr to listen on all interfaces |
 | `METRICS_PATH` |  path for metrics, default `/metrics` |
 | `SCRAPE_DELAY` | scrape delay in seconds, default `300` |
-| `CF_BATCH_SIZE` | cloudflare request zones batch size (1 - 10), default `10` |
 | `METRICS_DENYLIST` | (Optional) cloudflare-exporter metrics to not export, comma delimited list of cloudflare-exporter metrics. If not set, all metrics are exported |
 | `ZONE_<NAME>` |  `DEPRECATED since 0.0.5` (optional) Zone ID. Add zones you want to scrape by adding env vars in this format. You can find the zone ids in Cloudflare dashboards. |
+| `LOG_LEVEL` | Set loglevel. Options are error, warn, info, debug. default `error` |
 
 Corresponding flags:
 ```
@@ -67,8 +68,8 @@ Corresponding flags:
   -listen=":8080": listen on addr:port ( default :8080), omit addr to listen on all interfaces
   -metrics_path="/metrics": path for metrics, default /metrics
   -scrape_delay=300: scrape delay in seconds, defaults to 300
-  -cf_batch_size=10: cloudflare zones batch size (1-10)
   -metrics_denylist="": cloudflare-exporter metrics to not export, comma delimited list
+  -log_level="error": log level(error,warn,info,debug)
 ```
 
 Note: `ZONE_<name>` configuration is not supported as flag.

cloudflare.go

@@ -2,19 +2,38 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"strings"
+	"sync"
 	"time"
 
-	cloudflare "github.com/cloudflare/cloudflare-go"
+	cf "github.com/cloudflare/cloudflare-go/v4"
+	cfaccounts "github.com/cloudflare/cloudflare-go/v4/accounts"
+	cfload_balancers "github.com/cloudflare/cloudflare-go/v4/load_balancers"
+	cfrulesets "github.com/cloudflare/cloudflare-go/v4/rulesets"
+	cfzones "github.com/cloudflare/cloudflare-go/v4/zones"
 	"github.com/machinebox/graphql"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
+const (
+	freePlanId      = "0feeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
+	apiPerPageLimit = 1000
+	gqlQueryLimit   = 9999
+)
+
 var (
 	cfGraphQLEndpoint = "https://api.cloudflare.com/client/v4/graphql/"
+	gql               = &GQL{
+		Client: graphql.NewClient(cfGraphQLEndpoint),
+	}
 )
 
+type GQL struct {
+	Client *graphql.Client
+	Mu     sync.RWMutex
+}
+
 type cloudflareResponse struct {
 	Viewer struct {
 		Zones []zoneResp `json:"zones"`
@@ -250,62 +269,71 @@ type lbResp struct {
 	ZoneTag string `json:"zoneTag"`
 }
 
-func fetchZones() []cloudflare.Zone {
-	var api *cloudflare.API
-	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
+func fetchLoadblancerPools(account cfaccounts.Account) []cfload_balancers.Pool {
+	ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
+	defer cancel()
+	pools, err := cfclient.LoadBalancers.Pools.List(
+		ctx,
+		cfload_balancers.PoolListParams{
+			AccountID: cf.F(account.ID),
+		})
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
+		return nil
 	}
 
-	ctx := context.Background()
-	z, err := api.ListZones(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
+	return pools.Result
+}
+
+func fetchZones(accounts []cfaccounts.Account) []cfzones.Zone {
 
-	return z
+	var zones []cfzones.Zone
+
+	for _, account := range accounts {
+		ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
+		z, err := cfclient.Zones.List(ctx, cfzones.ZoneListParams{
+			Account: cf.F(cfzones.ZoneListParamsAccount{ID: cf.F(account.ID)}),
+			PerPage: cf.F(float64(apiPerPageLimit)),
+		})
+
+		if err != nil {
+			log.Error(err)
+			cancel()
+			continue
+		}
+		zones = append(zones, z.Result...)
+		cancel()
+	}
+	return zones
 }
 
 func fetchFirewallRules(zoneID string) map[string]string {
-	var api *cloudflare.API
-	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
-	if err != nil {
-		log.Fatal(err)
-	}
 
-	ctx := context.Background()
-	listOfRules, _, err := api.FirewallRules(ctx,
-		cloudflare.ZoneIdentifier(zoneID),
-		cloudflare.FirewallRuleListParams{})
+	ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
+	defer cancel()
+
+	listOfRulesets, err := cfclient.Rulesets.List(ctx, cfrulesets.RulesetListParams{
+		ZoneID: cf.F(zoneID),
+	})
 	if err != nil {
-		log.Fatal(err)
+		log.Errorf("ZoneID:%s, Err:%s", zoneID, err)
+		return map[string]string{}
 	}
-	firewallRulesMap := make(map[string]string)
 
-	for _, rule := range listOfRules {
-		firewallRulesMap[rule.ID] = rule.Description
-	}
+	firewallRulesMap := make(map[string]string)
 
-	listOfRulesets, err := api.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{})
-	if err != nil {
-		log.Fatal(err)
-	}
-	for _, rulesetDesc := range listOfRulesets {
-		if rulesetDesc.Phase == "http_request_firewall_managed" {
-			ruleset, err := api.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
+	for _, rulesetDesc := range listOfRulesets.Result {
+		if rulesetDesc.Phase == cfrulesets.PhaseHTTPRequestFirewallManaged {
+			ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
+			ruleset, err := cfclient.Rulesets.Get(ctx, rulesetDesc.ID, cfrulesets.RulesetGetParams{
+				ZoneID: cf.F(zoneID),
+			})
 			if err != nil {
-				log.Fatal(err)
+				log.Errorf("ZoneID:%s, RulesetID:%s, Err:%s", zoneID, rulesetDesc.ID, err)
+				cancel()
+				continue
 			}
+			cancel()
 			for _, rule := range ruleset.Rules {
 				firewallRulesMap[rule.ID] = rule.Description
 			}
@@ -315,25 +343,18 @@ func fetchFirewallRules(zoneID string) map[string]string {
 	return firewallRulesMap
 }
 
-func fetchAccounts() []cloudflare.Account {
-	var api *cloudflare.API
+func fetchAccounts() []cfaccounts.Account {
 	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
-	if err != nil {
-		log.Fatal(err)
-	}
 
-	ctx := context.Background()
-	a, _, err := api.Accounts(ctx, cloudflare.AccountsListParams{PaginationOptions: cloudflare.PaginationOptions{PerPage: 100}})
+	ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
+	defer cancel()
+	a, err := cfclient.Accounts.List(ctx, cfaccounts.AccountListParams{PerPage: cf.F(float64(apiPerPageLimit))})
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
+		return []cfaccounts.Account{}
 	}
 
-	return a
+	return a.Result
 }
 
 func fetchZoneTotals(zoneIDs []string) (*cloudflareResponse, error) {
@@ -445,16 +466,17 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) {
 		request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
 		request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
 	}
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 	request.Var("zoneIDs", zoneIDs)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 
 	var resp cloudflareResponse
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
@@ -501,15 +523,16 @@ func fetchColoTotals(zoneIDs []string) (*cloudflareResponseColo, error) {
 		request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
 		request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
 	}
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 	request.Var("zoneIDs", zoneIDs)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseColo
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
@@ -561,15 +584,16 @@ func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) {
 		request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
 		request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
 	}
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 	request.Var("accountID", accountID)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseAccts
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
@@ -638,15 +662,16 @@ func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) {
 		request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
 		request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
 	}
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 	request.Var("zoneIDs", zoneIDs)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLb
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
@@ -691,14 +716,15 @@ func fetchLogpushAccount(accountID string) (*cloudflareResponseLogpushAccount, e
 	}
 
 	request.Var("accountID", accountID)
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLogpushAccount
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
@@ -743,22 +769,23 @@ func fetchLogpushZone(zoneIDs []string) (*cloudflareResponseLogpushZone, error)
 	}
 
 	request.Var("zoneIDs", zoneIDs)
-	request.Var("limit", 9999)
+	request.Var("limit", gqlQueryLimit)
 	request.Var("maxtime", now)
 	request.Var("mintime", now1mAgo)
 
 	ctx := context.Background()
-	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLogpushZone
-	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+	gql.Mu.RLock()
+	defer gql.Mu.RUnlock()
+	if err := gql.Client.Run(ctx, request, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}
 
 	return &resp, nil
 }
 
-func findZoneAccountName(zones []cloudflare.Zone, ID string) (string, string) {
+func findZoneAccountName(zones []cfzones.Zone, ID string) (string, string) {
 	for _, z := range zones {
 		if z.ID == ID {
 			return z.Name, strings.ToLower(strings.ReplaceAll(z.Account.Name, " ", "-"))
@@ -768,7 +795,7 @@ func findZoneAccountName(zones []cloudflare.Zone, ID string) (string, string) {
 	return "", ""
 }
 
-func extractZoneIDs(zones []cloudflare.Zone) []string {
+func extractZoneIDs(zones []cfzones.Zone) []string {
 	var IDs []string
 
 	for _, z := range zones {
@@ -778,11 +805,29 @@ func extractZoneIDs(zones []cloudflare.Zone) []string {
 	return IDs
 }
 
-func filterNonFreePlanZones(zones []cloudflare.Zone) (filteredZones []cloudflare.Zone) {
+func filterNonFreePlanZones(zones []cfzones.Zone) (filteredZones []cfzones.Zone) {
+
+	var zoneIDs []string
+
 	for _, z := range zones {
-		if z.Plan.ZonePlanCommon.ID != "0feeeeeeeeeeeeeeeeeeeeeeeeeeeeee" {
+		extraFields, err := extractExtraFields(z.JSON.ExtraFields["plan"].Raw())
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+		if extraFields["id"] == freePlanId {
+			continue
+		}
+		if !contains(zoneIDs, z.ID) {
+			zoneIDs = append(zoneIDs, z.ID)
 			filteredZones = append(filteredZones, z)
 		}
 	}
 	return
 }
+
+func extractExtraFields(fields string) (map[string]interface{}, error) {
+	var extraFields map[string]interface{}
+	err := json.Unmarshal([]byte(fields), &extraFields)
+	return extraFields, err
+}