feat: Ensure new common configurations from the file are applied (#5230)

After upgrading EdgeX, new configurations may be introduced. Ensure new configurations from the YAML file are pushed to the core-keeper.

Signed-off-by: bruce <[email protected]>

Author: weichou1229

go.mod

@@ -5,8 +5,7 @@ go 1.23.7
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/eclipse/paho.mqtt.golang v1.5.0
-	github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.30
-	github.com/edgexfoundry/go-mod-configuration/v4 v4.1.0-dev.12
+	github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.31
 	github.com/edgexfoundry/go-mod-core-contracts/v4 v4.1.0-dev.12
 	github.com/edgexfoundry/go-mod-messaging/v4 v4.1.0-dev.13
 	github.com/edgexfoundry/go-mod-secrets/v4 v4.1.0-dev.4
@@ -32,6 +31,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/edgexfoundry/go-mod-configuration/v4 v4.1.0-dev.12 // indirect
 	github.com/edgexfoundry/go-mod-registry/v4 v4.1.0-dev.4 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect

go.sum

@@ -72,8 +72,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
 github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
-github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.30 h1:3Onz1+oxAlRlMchjfL0bO+XTP63lMqmlpjobQNP8Bn0=
-github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.30/go.mod h1:sQKOtdQlVuRzo06mDgNbJeyT/9QV94LlaY5wO1MsycU=
+github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.31 h1:YTGd985lXEySEt76d/yXD8BaQAFOZFlBS7VfBJVLUIY=
+github.com/edgexfoundry/go-mod-bootstrap/v4 v4.1.0-dev.31/go.mod h1:sQKOtdQlVuRzo06mDgNbJeyT/9QV94LlaY5wO1MsycU=
 github.com/edgexfoundry/go-mod-configuration/v4 v4.1.0-dev.12 h1:sW0dy9juPYPOwgwZAq1riDhJ96ivZBV9bdn+qWeeWO0=
 github.com/edgexfoundry/go-mod-configuration/v4 v4.1.0-dev.12/go.mod h1:8av7ex+XGeQHXQe/qyCdZ6g9AYWAYGRkkeFK0TKbiOQ=
 github.com/edgexfoundry/go-mod-core-contracts/v4 v4.1.0-dev.12 h1:y5DmnfDWSzkil3n/mOM3geNlObQw59U75901R4s0a94=

internal/core/common_config/main.go

@@ -1,5 +1,6 @@
 /*******************************************************************************
  * Copyright 2023 Intel Corporation
+ * Copyright 2025 IOTech Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
  * in compliance with the License. You may obtain a copy of the License at
@@ -17,25 +18,25 @@ package common_config
 import (
 	"context"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"net/http"
 	"os"
 	"os/signal"
-	"sort"
 	"sync"
 	"syscall"
 
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/config"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/container"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/environment"
+	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/file"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/flags"
+	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/interfaces"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/secret"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/startup"
 	"github.com/edgexfoundry/go-mod-bootstrap/v4/di"
-	"github.com/edgexfoundry/go-mod-configuration/v4/configuration"
 	"github.com/edgexfoundry/go-mod-core-contracts/v4/clients/logger"
 	"github.com/edgexfoundry/go-mod-core-contracts/v4/common"
 	"github.com/edgexfoundry/go-mod-core-contracts/v4/models"
-	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -96,37 +97,39 @@ func Main(ctx context.Context, cancel context.CancelFunc, args []string) {
 		os.Exit(1)
 	}
 
-	hasConfig := false
-	hasConfigSuccess := false
-	for startupTimer.HasNotElapsed() {
-		// check to see if the configuration exists in the config provider
-		hasConfig, err = configClient.HasConfiguration()
-		if err == nil {
-			hasConfigSuccess = true
-			break
-		}
-
-		lc.Warnf("Unable to determine if common configuration exists in the provider, will try again: %v", err)
-		startupTimer.SleepForInterval()
+	// push not done flag to configClient
+	err = configClient.PutConfigurationValue(commonConfigDone, []byte(common.ValueFalse))
+	if err != nil {
+		lc.Errorf("failed to push %s on startup: %s", commonConfigDone, err.Error())
+		os.Exit(1)
 	}
 
-	if !hasConfigSuccess {
-		lc.Errorf("failed to determine if common configuration exists in the provider: %s", err.Error())
+	yamlFile := config.GetConfigFileLocation(lc, f)
+	lc.Infof("Using common configuration from %s", yamlFile)
+
+	configMap, err := loadConfigMapFromYamlFile(yamlFile, secretProvider, lc)
+	if err != nil {
+		lc.Errorf("Failed to load %s : %s", yamlFile, err.Error())
+		os.Exit(1)
+	}
+	configMap, err = applyEnvOverrides(configMap, lc)
+	if err != nil {
+		lc.Errorf("Failed to apply env overrides to common configuration: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// load the yaml file and push it using the config client
-	if !hasConfig || getOverwriteConfig(f, lc) {
-		lc.Info("Pushing common configuration. It doesn't exists or overwrite flag is set")
+	// PutConfigurationMap func will check each keys and put the config value if not exist or override is set
+	err = configClient.PutConfigurationMap(configMap, getOverwriteConfig(f, lc))
+	if err != nil {
+		lc.Errorf("Failed to put configuration: %s", err.Error())
+		os.Exit(1)
+	}
 
-		yamlFile := config.GetConfigFileLocation(lc, f)
-		err = pushConfiguration(lc, yamlFile, configClient)
-		if err != nil {
-			lc.Error(err.Error())
-			os.Exit(1)
-		}
-	} else {
-		lc.Info("Skipped pushing common configuration. It already exists and overwrite flag not set")
+	// push done flag to config client
+	err = configClient.PutConfigurationValue(commonConfigDone, []byte(common.ValueTrue))
+	if err != nil {
+		lc.Errorf("Failed to push %s on completion: %s", commonConfigDone, err.Error())
+		os.Exit(1)
 	}
 
 	lc.Info("Core Common Config exiting")
@@ -155,96 +158,30 @@ func translateInterruptToCancel(ctx context.Context, wg *sync.WaitGroup, cancel
 	}()
 }
 
-func pushConfiguration(lc logger.LoggingClient, yamlFile string, configClient configuration.Client) error {
-	// push not done flag to configClient
-	err := configClient.PutConfigurationValue(commonConfigDone, []byte("false"))
-	if err != nil {
-		return fmt.Errorf("failed to push %s on startup: %s", commonConfigDone, err.Error())
-	}
-	lc.Infof("Using common configuration from %s", yamlFile)
-	contents, err := os.ReadFile(yamlFile)
-	if err != nil {
-		return fmt.Errorf("failed to read common configuration file %s: %s", yamlFile, err.Error())
-	}
-
-	data := make(map[string]interface{})
-	kv := make(map[string]interface{})
-
-	err = yaml.Unmarshal(contents, &data)
-	if err != nil {
-		return fmt.Errorf("failed to unmarshall common configuration file %s: %s", yamlFile, err.Error())
-	}
-
-	kv = buildKeyValues(data, kv, "")
-
-	kv, err = applyEnvOverrides(kv, lc)
-	if err != nil {
-		return fmt.Errorf("failed to apply env overrides to common configuration: %s", err.Error())
-	}
-
-	keys := make([]string, 0, len(kv))
-
-	for k := range kv {
-		keys = append(keys, k)
-	}
-
-	sort.Strings(keys)
-
-	for _, k := range keys {
-		v := kv[k]
-		// Push key/value into Configuration Provider if it is not empty
-		if v != nil {
-			err = configClient.PutConfigurationValue(k, []byte(fmt.Sprint(v)))
-		}
-		if err != nil {
-			return fmt.Errorf("failed to push common configuration key %s with value %v: %s", k, v, err.Error())
-		}
-	}
+func applyEnvOverrides(keyValues map[string]any, lc logger.LoggingClient) (map[string]any, error) {
+	env := environment.NewVariables(lc)
 
-	// push done flag to config client
-	err = configClient.PutConfigurationValue(commonConfigDone, []byte("true"))
+	overrideCount, err := env.OverrideConfigMapValues(keyValues)
 	if err != nil {
-		return fmt.Errorf("failed to push %s on completion: %s", commonConfigDone, err.Error())
+		return nil, err
 	}
 
-	lc.Info("Common configuration has been pushed to into Configuration Provider with overrides applied")
+	lc.Infof("Common configuration loaded from file with %d overrides applied", overrideCount)
 
-	return nil
+	return keyValues, nil
 }
 
-// buildKeyValues is a helper function to parse the configuration yaml file contents
-func buildKeyValues(data map[string]interface{}, kv map[string]interface{}, origKey string) map[string]interface{} {
-	key := origKey
-	for k, v := range data {
-		if len(key) == 0 {
-			key = fmt.Sprint(k)
-		} else {
-			key = fmt.Sprintf("%s/%s", key, k)
-		}
-
-		vdata, ok := v.(map[string]interface{})
-		if !ok {
-			kv[key] = v
-			key = origKey
-			continue
-		}
-
-		kv = buildKeyValues(vdata, kv, key)
-		key = origKey
+func loadConfigMapFromYamlFile(yamlFile string, secretProvider interfaces.SecretProvider, lc logger.LoggingClient) (map[string]any, error) {
+	contents, err := file.Load(yamlFile, secretProvider, lc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read configuration file %s: %s", yamlFile, err.Error())
 	}
 
-	return kv
-}
-
-func applyEnvOverrides(keyValues map[string]any, lc logger.LoggingClient) (map[string]any, error) {
-	env := environment.NewVariables(lc)
+	data := make(map[string]any)
 
-	overrideCount, err := env.OverrideConfigMapValues(keyValues)
+	err = yaml.Unmarshal(contents, &data)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to unmarshall yaml configuration: %s", err.Error())
 	}
-
-	lc.Infof("Common configuration loaded from file with %d overrides applied", overrideCount)
-
-	return keyValues, nil
+	return data, nil
 }