[ISSUE #10377] Nacos client observability enhancement - trace part

client/pom.xml

@@ -32,6 +32,19 @@
     <url>https://nacos.io</url>
     <description>Nacos client pom.xml file</description>
 
+    <!--  Manage the version of libraries in OpenTelemetry  -->
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-bom</artifactId>
+                <version>${io.opentelemetry.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
 
         <dependency>
@@ -126,6 +139,26 @@
             <groupId>io.micrometer</groupId>
             <artifactId>micrometer-registry-otlp</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-semconv</artifactId>
+            <version>${io.opentelemetry.version}-alpha</version>
+        </dependency>
 
     </dependencies>
 

client/src/main/java/com/alibaba/nacos/client/config/NacosConfigService.java

@@ -26,6 +26,7 @@
 import com.alibaba.nacos.client.config.filter.impl.ConfigFilterChainManager;
 import com.alibaba.nacos.client.config.filter.impl.ConfigRequest;
 import com.alibaba.nacos.client.config.filter.impl.ConfigResponse;
+import com.alibaba.nacos.client.config.http.HttpAgent;
 import com.alibaba.nacos.client.config.http.ServerHttpAgent;
 import com.alibaba.nacos.client.config.impl.ClientWorker;
 import com.alibaba.nacos.client.config.impl.LocalConfigInfoProcessor;
@@ -34,10 +35,19 @@
 import com.alibaba.nacos.client.config.utils.ContentUtils;
 import com.alibaba.nacos.client.config.utils.ParamUtils;
 import com.alibaba.nacos.client.env.NacosClientProperties;
+import com.alibaba.nacos.client.monitor.TraceDynamicProxy;
+import com.alibaba.nacos.client.monitor.config.ClientWorkerTraceProxy;
+import com.alibaba.nacos.common.constant.NacosSemanticAttributes;
+import com.alibaba.nacos.client.monitor.config.ConfigTrace;
 import com.alibaba.nacos.client.utils.LogUtils;
 import com.alibaba.nacos.client.utils.ParamUtil;
 import com.alibaba.nacos.client.utils.ValidatorUtils;
 import com.alibaba.nacos.common.utils.StringUtils;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanBuilder;
+import io.opentelemetry.api.trace.StatusCode;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
 import org.slf4j.Logger;
 
 import java.util.Arrays;
@@ -61,12 +71,12 @@ public class NacosConfigService implements ConfigService {
      * will be deleted in 2.0 later versions
      */
     @Deprecated
-    ServerHttpAgent agent = null;
+    HttpAgent agent;
 
     /**
      * long polling.
      */
-    private final ClientWorker worker;
+    private final ClientWorkerTraceProxy worker;
 
     private String namespace;
 
@@ -81,9 +91,10 @@ public NacosConfigService(Properties properties) throws NacosException {
         ServerListManager serverListManager = new ServerListManager(clientProperties);
         serverListManager.start();
 
-        this.worker = new ClientWorker(this.configFilterChainManager, serverListManager, clientProperties);
+        this.worker = TraceDynamicProxy.getClientWorkerTraceProxy(
+                new ClientWorker(this.configFilterChainManager, serverListManager, clientProperties));
         // will be deleted in 2.0 later versions
-        agent = new ServerHttpAgent(serverListManager);
+        agent = TraceDynamicProxy.getHttpAgentTraceProxy(new ServerHttpAgent(serverListManager));
 
     }
 
@@ -102,8 +113,8 @@ public String getConfigAndSignListener(String dataId, String group, long timeout
             throws NacosException {
         group = StringUtils.isBlank(group) ? Constants.DEFAULT_GROUP : group.trim();
 
-        ConfigResponse configResponse = worker.getAgent()
-                .queryConfig(dataId, group, worker.getAgent().getTenant(), timeoutMs, false);
+        ConfigResponse configResponse = worker.getServerConfig(dataId, group, worker.getAgentTenant(), timeoutMs,
+                false);
 
         String content = configResponse.getContent();
         String encryptedDataKey = configResponse.getEncryptedDataKey();
@@ -171,14 +182,15 @@ private String getConfigInner(String tenant, String dataId, String group, long t
         // but is maintained by user.
         // This is designed for certain scenario like client emergency reboot,
         // changing config needed in the same time, while nacos server is down.
-        String content = LocalConfigInfoProcessor.getFailover(worker.getAgentName(), dataId, group, tenant);
+        String content = getFailoverWithTrace(worker.getAgentName(), dataId, group, tenant, false);
 
         if (content != null) {
             LOGGER.warn("[{}] [get-config] get failover ok, dataId={}, group={}, tenant={}, config={}",
                     worker.getAgentName(), dataId, group, tenant, ContentUtils.truncateContent(content));
             cr.setContent(content);
-            String encryptedDataKey = LocalEncryptedDataKeyProcessor.getEncryptDataKeyFailover(agent.getName(), dataId,
-                    group, tenant);
+
+            String encryptedDataKey = getFailoverWithTrace(agent.getName(), dataId, group, tenant, true);
+
             cr.setEncryptedDataKey(encryptedDataKey);
             configFilterChainManager.doFilter(null, cr);
             content = cr.getContent();
@@ -203,21 +215,142 @@ private String getConfigInner(String tenant, String dataId, String group, long t
                     worker.getAgentName(), dataId, group, tenant, ioe.toString());
         }
 
-        content = LocalConfigInfoProcessor.getSnapshot(worker.getAgentName(), dataId, group, tenant);
+        content = getSnapshotWithTrace(worker.getAgentName(), dataId, group, tenant, false);
 
         if (content != null) {
             LOGGER.warn("[{}] [get-config] get snapshot ok, dataId={}, group={}, tenant={}, config={}",
                     worker.getAgentName(), dataId, group, tenant, ContentUtils.truncateContent(content));
         }
         cr.setContent(content);
-        String encryptedDataKey = LocalEncryptedDataKeyProcessor.getEncryptDataKeySnapshot(agent.getName(), dataId,
-                group, tenant);
+
+        String encryptedDataKey = getSnapshotWithTrace(agent.getName(), dataId, group, tenant, true);
+
         cr.setEncryptedDataKey(encryptedDataKey);
         configFilterChainManager.doFilter(null, cr);
         content = cr.getContent();
         return content;
     }
 
+    /**
+     * Get failover with trace.
+     *
+     * <p>Didn't use dynamic proxy here because of calling static method.
+     *
+     * @param envName     env name
+     * @param dataId      data id
+     * @param group       group
+     * @param tenant      tenant
+     * @param isEncrypted is encrypted
+     * @return content
+     */
+    private String getFailoverWithTrace(String envName, String dataId, String group, String tenant,
+            boolean isEncrypted) {
+
+        SpanBuilder spanBuilder;
+        if (isEncrypted) {
+            spanBuilder = ConfigTrace.getClientConfigServiceSpanBuilder("getEncryptDataKeyFailover");
+            spanBuilder.setAttribute(SemanticAttributes.CODE_NAMESPACE, LocalEncryptedDataKeyProcessor.class.getName());
+        } else {
+            spanBuilder = ConfigTrace.getClientConfigServiceSpanBuilder("getFailover");
+            spanBuilder.setAttribute(SemanticAttributes.CODE_NAMESPACE, LocalConfigInfoProcessor.class.getName());
+        }
+
+        Span span = setSpanLocalConfigInfoAttributes(envName, dataId, group, tenant, spanBuilder);
+
+        String content;
+        try (Scope ignored = span.makeCurrent()) {
+
+            if (isEncrypted) {
+                content = LocalEncryptedDataKeyProcessor.getEncryptDataKeyFailover(envName, dataId, group, tenant);
+            } else {
+                content = LocalConfigInfoProcessor.getFailover(envName, dataId, group, tenant);
+            }
+
+            if (span.isRecording()) {
+                if (content == null) {
+                    span.setStatus(StatusCode.ERROR, "Get failover failed");
+                } else {
+                    span.setStatus(StatusCode.OK, "Get failover ok");
+                    span.setAttribute(NacosSemanticAttributes.CONTENT, content);
+                }
+            }
+
+        } catch (Throwable e) {
+            span.recordException(e);
+            span.setStatus(StatusCode.ERROR, e.getClass().getSimpleName());
+            throw e;
+        } finally {
+            span.end();
+        }
+
+        return content;
+    }
+
+    /**
+     * Get snapshot with trace.
+     *
+     * <p>Didn't use dynamic proxy here because of calling static method.
+     *
+     * @param envName     env name
+     * @param dataId      data id
+     * @param group       group
+     * @param tenant      tenant
+     * @param isEncrypted is encrypted
+     * @return content
+     */
+    private String getSnapshotWithTrace(String envName, String dataId, String group, String tenant,
+            boolean isEncrypted) {
+
+        SpanBuilder spanBuilder;
+        if (isEncrypted) {
+            spanBuilder = ConfigTrace.getClientConfigServiceSpanBuilder("getEncryptDataKeySnapshot");
+            spanBuilder.setAttribute(SemanticAttributes.CODE_NAMESPACE, LocalEncryptedDataKeyProcessor.class.getName());
+        } else {
+            spanBuilder = ConfigTrace.getClientConfigServiceSpanBuilder("getSnapshot");
+            spanBuilder.setAttribute(SemanticAttributes.CODE_NAMESPACE, LocalConfigInfoProcessor.class.getName());
+        }
+
+        Span span = setSpanLocalConfigInfoAttributes(envName, dataId, group, tenant, spanBuilder);
+
+        String content;
+        try (Scope ignored = span.makeCurrent()) {
+
+            if (isEncrypted) {
+                content = LocalEncryptedDataKeyProcessor.getEncryptDataKeySnapshot(envName, dataId, group, tenant);
+            } else {
+                content = LocalConfigInfoProcessor.getSnapshot(envName, dataId, group, tenant);
+            }
+
+            if (span.isRecording()) {
+                if (content == null) {
+                    span.setStatus(StatusCode.ERROR, "Get snapshot failed");
+                } else {
+                    span.setStatus(StatusCode.OK, "Get snapshot ok");
+                    span.setAttribute(NacosSemanticAttributes.CONTENT, content);
+                }
+            }
+
+        } catch (Throwable e) {
+            span.recordException(e);
+            span.setStatus(StatusCode.ERROR, e.getClass().getSimpleName());
+            throw e;
+        } finally {
+            span.end();
+        }
+        return content;
+    }
+
+    private Span setSpanLocalConfigInfoAttributes(String envName, String dataId, String group, String tenant,
+            SpanBuilder spanBuilder) {
+        spanBuilder.setAttribute(NacosSemanticAttributes.AGENT_NAME, envName);
+        spanBuilder.setAttribute(NacosSemanticAttributes.DATA_ID, dataId);
+        spanBuilder.setAttribute(NacosSemanticAttributes.GROUP, group);
+        spanBuilder.setAttribute(NacosSemanticAttributes.TENANT, tenant);
+        spanBuilder.setAttribute(NacosSemanticAttributes.NAMESPACE, namespace);
+
+        return spanBuilder.startSpan();
+    }
+
     private String blank2defaultGroup(String group) {
         return (StringUtils.isBlank(group)) ? Constants.DEFAULT_GROUP : group.trim();
     }

client/src/main/java/com/alibaba/nacos/client/config/http/MetricsHttpAgent.java

@@ -20,6 +20,7 @@
 import com.alibaba.nacos.client.monitor.config.ConfigMetrics;
 import com.alibaba.nacos.common.http.HttpRestResult;
 
+import java.net.HttpURLConnection;
 import java.util.Date;
 import java.util.Map;
 
@@ -124,5 +125,13 @@ public String getEncode() {
     public void shutdown() throws NacosException {
         httpAgent.shutdown();
     }
+
+    private boolean isFail(HttpRestResult<String> result) {
+        return result.getCode() == HttpURLConnection.HTTP_INTERNAL_ERROR
+                || result.getCode() == HttpURLConnection.HTTP_BAD_GATEWAY
+                || result.getCode() == HttpURLConnection.HTTP_UNAVAILABLE
+                || result.getCode() == HttpURLConnection.HTTP_NOT_FOUND;
+    }
+
 }
 

client/src/main/java/com/alibaba/nacos/client/config/http/ServerHttpAgent.java

@@ -246,7 +246,7 @@ private boolean isFail(HttpRestResult<String> result) {
                 || result.getCode() == HttpURLConnection.HTTP_NOT_FOUND;
     }
 
-    public static String getAppname() {
+    public static String getAppName() {
         return ParamUtil.getAppName();
     }