ref: add rule intervals per rule type

Instead of a singular global default, now a rule_group interval can be
set for every individual type of rule_group Sloth generates.

The generic, `interval:all` rule will also stay and can "fill in" any missing per-rule
group defaults. Along with the default behavior of doing nothing if no
`interval` is specified.

Author: wbollock

internal/prometheus/model.go

@@ -39,17 +39,20 @@ type AlertMeta struct {
 
 // SLO represents a service level objective configuration.
 type SLO struct {
-	ID                string `validate:"required,name"`
-	Name              string `validate:"required,name"`
-	Description       string
-	Service           string `validate:"required,name"`
-	RuleGroupInterval string
-	SLI               SLI               `validate:"required"`
-	TimeWindow        time.Duration     `validate:"required"`
-	Objective         float64           `validate:"gt=0,lte=100"`
-	Labels            map[string]string `validate:"dive,keys,prom_label_key,endkeys,required,prom_label_value"`
-	PageAlertMeta     AlertMeta
-	TicketAlertMeta   AlertMeta
+	ID                    string `validate:"required,name"`
+	Name                  string `validate:"required,name"`
+	Description           string
+	Service               string            `validate:"required,name"`
+	RuleGroupInterval     time.Duration     `validate:"time"`
+	SLIErrorRulesInterval time.Duration     `validate:"time"`
+	MetadataRulesInterval time.Duration     `validate:"time"`
+	AlertRulesInterval    time.Duration     `validate:"time"`
+	SLI                   SLI               `validate:"required"`
+	TimeWindow            time.Duration     `validate:"required"`
+	Objective             float64           `validate:"gt=0,lte=100"`
+	Labels                map[string]string `validate:"dive,keys,prom_label_key,endkeys,required,prom_label_value"`
+	PageAlertMeta         AlertMeta
+	TicketAlertMeta       AlertMeta
 }
 
 type SLOGroup struct {
@@ -87,6 +90,7 @@ var modelSpecValidate = func() *validator.Validate {
 	mustRegisterValidation(v, "name", validateName)
 	mustRegisterValidation(v, "required_if_enabled", validateRequiredEnabledAlertName)
 	mustRegisterValidation(v, "template_vars", validateTemplateVars)
+	mustRegisterValidation(v, "time", validateTime)
 	v.RegisterStructValidation(validateOneSLI, SLI{})
 	v.RegisterStructValidation(validateSLOGroup, SLOGroup{})
 	v.RegisterStructValidation(validateSLIEvents, SLIEvents{})
@@ -182,6 +186,18 @@ func validateName(fl validator.FieldLevel) bool {
 	return nameRegexp.MatchString(s)
 }
 
+// validateTime implements validator.CustomTypeFunc by validating
+// a time duration.
+func validateTime(fl validator.FieldLevel) bool {
+	s, ok := fl.Field().Interface().(time.Duration)
+	if !ok {
+		return false
+	}
+
+	_, err := time.ParseDuration(s.String())
+	return err == nil
+}
+
 func validateRequiredEnabledAlertName(fl validator.FieldLevel) bool {
 	alertMeta, ok := fl.Parent().Interface().(AlertMeta)
 	if !ok {

internal/prometheus/spec.go

@@ -69,16 +69,19 @@ func (y YAMLSpecLoader) mapSpecToModel(ctx context.Context, spec prometheusv1.Sp
 	models := make([]SLO, 0, len(spec.SLOs))
 	for _, specSLO := range spec.SLOs {
 		slo := SLO{
-			ID:                fmt.Sprintf("%s-%s", spec.Service, specSLO.Name),
-			RuleGroupInterval: specSLO.RuleGroupInterval,
-			Name:              specSLO.Name,
-			Description:       specSLO.Description,
-			Service:           spec.Service,
-			TimeWindow:        y.windowPeriod,
-			Objective:         specSLO.Objective,
-			Labels:            mergeLabels(spec.Labels, specSLO.Labels),
-			PageAlertMeta:     AlertMeta{Disable: true},
-			TicketAlertMeta:   AlertMeta{Disable: true},
+			ID:                    fmt.Sprintf("%s-%s", spec.Service, specSLO.Name),
+			RuleGroupInterval:     specSLO.Interval.RuleGroupInterval,
+			SLIErrorRulesInterval: specSLO.Interval.SLIErrorRulesInterval,
+			MetadataRulesInterval: specSLO.Interval.MetadataRulesInterval,
+			AlertRulesInterval:    specSLO.Interval.AlertRulesInterval,
+			Name:                  specSLO.Name,
+			Description:           specSLO.Description,
+			Service:               spec.Service,
+			TimeWindow:            y.windowPeriod,
+			Objective:             specSLO.Objective,
+			Labels:                mergeLabels(spec.Labels, specSLO.Labels),
+			PageAlertMeta:         AlertMeta{Disable: true},
+			TicketAlertMeta:       AlertMeta{Disable: true},
 		}
 
 		// Set SLIs.

internal/prometheus/storage.go

@@ -49,9 +49,19 @@ func (i IOWriterGroupedRulesYAMLRepo) StoreSLOs(ctx context.Context, slos []Stor
 	ruleGroups := ruleGroupsYAMLv2{}
 	for _, slo := range slos {
 		if len(slo.Rules.SLIErrorRecRules) > 0 {
-			if slo.SLO.RuleGroupInterval != "" {
 
-				ruleGroupIntervalDuration, err := prommodel.ParseDuration(slo.SLO.RuleGroupInterval)
+			// 0s is default empty string value for time.Duration
+			if slo.SLO.RuleGroupInterval.String() != "0s" || slo.SLO.SLIErrorRulesInterval.String() != "0s" {
+				var ruleGroupIntervalDuration prommodel.Duration
+				var err error
+
+				// if we have a valid meta rule rule_group interval, use that first and overwrite any generic ones
+				if slo.SLO.SLIErrorRulesInterval.String() != "0s" {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.SLIErrorRulesInterval.String())
+				} else {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.RuleGroupInterval.String())
+				}
+
 				if err != nil {
 					return fmt.Errorf("could not parse rule_group interval duration %w", err)
 				}
@@ -71,9 +81,18 @@ func (i IOWriterGroupedRulesYAMLRepo) StoreSLOs(ctx context.Context, slos []Stor
 		}
 
 		if len(slo.Rules.MetadataRecRules) > 0 {
-			if slo.SLO.RuleGroupInterval != "" {
+			// if either of these aren't empty we'll be adding a custom rule interval
+			if slo.SLO.RuleGroupInterval.String() != "0s" || slo.SLO.MetadataRulesInterval.String() != "0s" {
+				var ruleGroupIntervalDuration prommodel.Duration
+				var err error
+
+				// if we have a valid meta rule rule_group interval, use that firs
+				if slo.SLO.MetadataRulesInterval.String() != "0s" {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.MetadataRulesInterval.String())
+				} else {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.RuleGroupInterval.String())
+				}
 
-				ruleGroupIntervalDuration, err := prommodel.ParseDuration(slo.SLO.RuleGroupInterval)
 				if err != nil {
 					return fmt.Errorf("could not parse rule_group interval duration %w", err)
 				}
@@ -92,9 +111,17 @@ func (i IOWriterGroupedRulesYAMLRepo) StoreSLOs(ctx context.Context, slos []Stor
 		}
 
 		if len(slo.Rules.AlertRules) > 0 {
-			if slo.SLO.RuleGroupInterval != "" {
+			if slo.SLO.RuleGroupInterval.String() != "0s" || slo.SLO.AlertRulesInterval.String() != "0s" {
+				var ruleGroupIntervalDuration prommodel.Duration
+				var err error
+
+				// if we have a valid meta rule rule_group interval, use that firs
+				if slo.SLO.AlertRulesInterval.String() != "0s" {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.AlertRulesInterval.String())
+				} else {
+					ruleGroupIntervalDuration, err = prommodel.ParseDuration(slo.SLO.RuleGroupInterval.String())
+				}
 
-				ruleGroupIntervalDuration, err := prommodel.ParseDuration(slo.SLO.RuleGroupInterval)
 				if err != nil {
 					return fmt.Errorf("could not parse rule_group interval duration %w", err)
 				}

pkg/prometheus/api/v1/v1.go

@@ -54,6 +54,8 @@
 //	          disable: true
 package v1
 
+import "time"
+
 const Version = "prometheus/v1"
 
 //go:generate gomarkdoc -o ./README.md ./
@@ -89,8 +91,9 @@ type SLO struct {
 	// Alerting is the configuration with all the things related with the SLO
 	// alerts.
 	Alerting Alerting `yaml:"alerting"`
-	// RuleGroupInterval is an optional value for how often the Prometheus rule_group should be evaluated.
-	RuleGroupInterval string `yaml:"interval,omitempty"`
+	// Interval is the configuration for all things related to SLO rule_group intervals
+	// for specific rule groups and all rules.
+	Interval Interval `yaml:"interval,omitempty"`
 }
 
 // SLI will tell what is good or bad for the SLO.
@@ -150,6 +153,18 @@ type Alerting struct {
 	TicketAlert Alert `yaml:"ticket_alert,omitempty"`
 }
 
+type Interval struct {
+	// RuleGroupInterval is an optional value for how often the Prometheus rule_group should be evaluated.
+	// RuleGroupInterval string `yaml:"rulegroup_interval,omitempty"`
+	RuleGroupInterval time.Duration `yaml:"all,omitempty"`
+	// Otherwise, specify custom rule_group intervals for each set of recording rules.
+	// RuleGroupInterval will "fill-in" for any non-specified individual groups
+	// but individual group settings override RuleGroupInterval.
+	SLIErrorRulesInterval time.Duration `yaml:"slierror,omitempty"`
+	MetadataRulesInterval time.Duration `yaml:"metadata,omitempty"`
+	AlertRulesInterval    time.Duration `yaml:"alert,omitempty"`
+}
+
 // Alert configures specific SLO alert.
 type Alert struct {
 	// Disable disables the alert and makes Sloth not generating this alert. This