Merge branch 'master' into alejandro.gonzalez/add-http-route-play-in-iast

Author: jandro996

.github/CODEOWNERS

@@ -110,3 +110,9 @@
 /internal-api/src/main/java/datadog/trace/api/EndpointCheckpointer.java                     @DataDog/profiling-java
 /internal-api/src/main/java/datadog/trace/api/EndpointTracker.java                          @DataDog/profiling-java
 /dd-smoke-tests/profiling-integration-tests/                                                @DataDog/profiling-java
+
+# @DataDog/ml-observability
+dd-trace-api/src/main/java/datadog/trace/api/llmobs/ @DataDog/ml-observability
+dd-java-agent/agent-llmobs/                          @DataDog/ml-observability
+dd-trace-core/src/main/java/datadog/trace/llmobs/    @DataDog/ml-observability
+dd-trace-core/src/test/groovy/datadog/trace/llmobs/  @DataDog/ml-observability

.gitlab-ci.yml

@@ -119,14 +119,11 @@ default:
 
 .gitlab_base_ref_params: &gitlab_base_ref_params
   - |
-    # FIXME: Disabled until we find a way to not hit GitHub API rate limit
-    if false && [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then
-      export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
-      if [[ -n "$GIT_BASE_REF" ]]; then
-        export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
-      else
-        echo "Failed to find base ref for PR" >&2
-      fi
+    export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
+    if [[ -n "$GIT_BASE_REF" ]]; then
+      export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
+    else
+      echo "Failed to find base ref for PR" >&2
     fi
 
 .gradle_build: &gradle_build
@@ -149,6 +146,7 @@ default:
         - .gradle/caches
         - .gradle/notifications
       policy: $DEPENDENCY_CACHE_POLICY
+      unprotect: true
       fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
         - '$CI_SERVER_VERSION-base'
         - '$CI_SERVER_VERSION-lib'
@@ -158,15 +156,19 @@ default:
         - .gradle/$GRADLE_VERSION/executionHistory
         - workspace
       policy: $BUILD_CACHE_POLICY
+      unprotect: true
   before_script:
     - source .gitlab/gitlab-utils.sh
+    # Akka token added to SSM from https://account.akka.io/token
+    - export AKKA_REPO_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
     - mkdir -p .gradle
     - export GRADLE_USER_HOME=$(pwd)/.gradle
     - |
       # Don't put jvm args here as it will be picked up by child gradle processes used in tests
       cat << EOF > $GRADLE_USER_HOME/gradle.properties
       mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
       gradlePluginProxy=$GRADLE_PLUGIN_PROXY
+      akkaRepositoryToken=$AKKA_REPO_TOKEN
       EOF
     - |
       # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
@@ -198,11 +200,11 @@ pre-release-checks:
       allow_failure: false
   script:
     - |
-      SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
-      SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
+      MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
+      MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
       # See https://central.sonatype.org/publish/publish-portal-api/
       # 15e0cbbb-deff-421e-9e02-296a24d0cada is deployment, any deployment id listed in central  work, the idea is to check whether the token can authenticate
-      curl --request POST --include --fail https://central.sonatype.com/api/v1/publisher/status?id=15e0cbbb-deff-421e-9e02-296a24d0cada --header "Authorization: Bearer $(printf "$SONATYPE_USERNAME:$SONATYPE_PASSWORD" | base64)"
+      curl --request POST --include --fail https://central.sonatype.com/api/v1/publisher/status?id=15e0cbbb-deff-421e-9e02-296a24d0cada --header "Authorization: Bearer $(printf "$MAVEN_CENTRAL_USERNAME:$MAVEN_CENTRAL_PASSWORD" | base64)"
       if [ $? -ne 0 ]; then
         echo "Failed to authenticate against central. Check credentials, see https://datadoghq.atlassian.net/wiki/x/Oog5OgE"
         exit 1
@@ -766,8 +768,8 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
     UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
 
-# If the deploy_to_sonatype job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
-deploy_to_sonatype:
+# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
+deploy_to_maven_central:
   extends: .gradle_build
   stage: publish
   needs: [ build ]
@@ -784,8 +786,8 @@ deploy_to_sonatype:
     - when: manual
       allow_failure: true
   script:
-    - export SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
-    - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
+    - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
+    - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
     - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
@@ -803,11 +805,11 @@ deploy_artifacts_to_github:
       when: never
     - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
       when: on_success
-  # Requires the deploy_to_sonatype job to have run first (the UP-TO-DATE gradle check across jobs is broken)
+  # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken)
   # This will deploy the artifacts built from the publishToSonatype task to the GitHub release
   needs:
-    - job: deploy_to_sonatype
-      # The deploy_to_sonatype job is not run for release candidate versions
+    - job: deploy_to_maven_central
+      # The deploy_to_maven_central job is not run for release candidate versions
       optional: true
   script:
     - aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.gh_release_token --with-decryption --query "Parameter.Value" --out text > github-token.txt

.gitlab/benchmarks.yml

@@ -76,6 +76,8 @@ check-big-regressions:
   when: on_success
   tags: ["arch:amd64"]
   rules:
+    - if: '$POPULATE_CACHE'
+      when: never
     - if: '$CI_COMMIT_BRANCH !~ /^(master|release\/)/'
       when: on_success
     - when: never

.gitlab/find-gh-base-ref.sh

@@ -2,6 +2,18 @@
 # Determines the base branch for the current PR (if we are running in a PR).
 set -euo pipefail
 
+if [[ -n "${CI_COMMIT_BRANCH:-}" ]]; then
+  echo "CI_COMMIT_BRANCH is set to $CI_COMMIT_BRANCH" >&2
+else
+  echo "CI_COMMIT_BRANCH is not set, skipping base ref detection" >&2
+  exit 1
+fi
+
+if [[ $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then
+  echo "CI_COMMIT_BRANCH is a master or release branch, skipping base ref detection" >&2
+  exit 1
+fi
+
 CURRENT_HEAD_SHA="$(git rev-parse HEAD)"
 if [[ -z "${CURRENT_HEAD_SHA:-}" ]]; then
   echo "Failed to determine current HEAD SHA" >&2
@@ -33,81 +45,61 @@ if [[ -f $CACHE_PATH ]]; then
 fi
 
 # Happy path: if we're just one commit away from master, base ref is master.
-if [[ $(git log --pretty=oneline origin/master..HEAD | wc -l) -eq 1 ]]; then
+if [[ $(git rev-list --count origin/master..HEAD) -eq 1 ]]; then
   echo "We are just one commit away from master, base ref is master" >&2
   save_cache "master" "$CURRENT_HEAD_SHA"
   echo "master"
   exit 0
 fi
 
-# In GitLab: we have no reference to the base branch or even the PR number.
-# We have to find it from the current branch name, which is defined in
-# CI_COMMIT_REF_NAME.
-if [[ -z "${CI_COMMIT_REF_NAME}" ]]; then
-  echo "CI_COMMIT_REF_NAME is not set, not running in GitLab CI?" >&2
-  exit 1
-fi
+get_distance_from_merge_base() {
+  local candidate_base="$1"
+  local merge_base_sha
+  local distance
+  merge_base_sha=$(git merge-base "$candidate_base" HEAD)
+  distance=$(git rev-list --count "$merge_base_sha".."$CURRENT_HEAD_SHA")
+  echo "Distance from $candidate_base is $distance" >&2
+  echo "$distance"
+}
 
-# In GitLab, CI_PROJECT_NAME is set, otherwise, set it for testing.
-export CI_PROJECT_NAME="${CI_PROJECT_NAME:-dd-trace-java}"
+# Find the best base ref: the master/release branch whose merge base is closest to HEAD.
+# If there are multiple candidates (e.g. immediately after a release branch is created), we cannot
+# disambiguate and return an error.
+# NOTE: GitHub API is more robust for this task, but we hit rate limits.
+BEST_CANDIDATES=(origin/master)
+BEST_DISTANCE=$(get_distance_from_merge_base origin/master)
 
-if [[ -z "${GITHUB_TOKEN:-}" ]]; then
-  echo "GITHUB_TOKEN is not set, fetching from AWS SSM" >&2
-  if ! command -v aws >/dev/null 2>&1; then
-    echo "aws is not installed, please install it" >&2
-    exit 1
-  fi
-  set +e
-  GITHUB_TOKEN=$(aws ssm get-parameter --name "ci.$CI_PROJECT_NAME.gh_release_token" --with-decryption --query "Parameter.Value" --output text)
-  set -e
-  if [[ -z "${GITHUB_TOKEN:-}" ]]; then
-    echo "Failed to fetch GITHUB_TOKEN from AWS SSM" >&2
-    exit 1
-  fi
-  export GITHUB_TOKEN
+# If the current branch is not a project/ branch, project/ branches are candidates.
+# This accounts for the case when the project/ branch is being merged to master.
+if [[ ! "$CI_COMMIT_BRANCH" =~ ^project/.*$ ]]; then
+  mapfile -t CANDIDATE_BASES < <(git branch -a --sort=committerdate --format='%(refname:short)' --list 'origin/release/v*' --list 'origin/project/*' | tac)
+else
+  mapfile -t CANDIDATE_BASES < <(git branch -a --sort=committerdate --format='%(refname:short)' --list 'origin/release/v*' | tac)
 fi
 
-if ! command -v curl >/dev/null 2>&1; then
-  echo "curl is not installed, please install it" >&2
-  exit 1
-fi
+for candidate_base in "${CANDIDATE_BASES[@]}"; do
+  distance=$(get_distance_from_merge_base "$candidate_base")
+  if [[ $distance -lt $BEST_DISTANCE ]]; then
+    BEST_DISTANCE=$distance
+    BEST_CANDIDATES=("$candidate_base")
+  elif [[ $distance -eq $BEST_DISTANCE ]]; then
+    BEST_CANDIDATES+=("$candidate_base")
+  fi
+done
 
-if ! command -v jq >/dev/null 2>&1; then
-  echo "jq is not installed, please install it" >&2
-  exit 1
+if [[ ${#BEST_CANDIDATES[@]} -eq 1 ]]; then
+  # Remote the origin/ prefix
+  base_ref="${BEST_CANDIDATES[0]#origin/}"
+  echo "Base ref is ${base_ref}" >&2
+  save_cache "${base_ref}" "$CURRENT_HEAD_SHA"
+  echo "${base_ref}"
+  exit 0
 fi
 
-while true; do
-  set +e
-  PR_DATA=$(curl \
-    -XGET \
-    --silent \
-    --include \
-    --fail-with-body \
-    -H 'Accept: application/vnd.github+json' \
-    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
-    -H "X-GitHub-Api-Version: 2022-11-28" \
-    "https://api.github.com/repos/datadog/dd-trace-java/pulls?head=DataDog:${CI_COMMIT_REF_NAME}&sort=updated&direction=desc")
-  exit_code=$?
-  set -e
-  if [[ ${exit_code} -eq 0 ]]; then
-    PR_NUMBER=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].number')
-    PR_BASE_REF=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].base.ref')
-    if [[ -n "${PR_BASE_REF:-}" ]]; then
-      echo "PR is https://github.com/datadog/dd-trace-java/pull/${PR_NUMBER} and base ref is ${PR_BASE_REF}">&2
-      save_cache "${PR_BASE_REF}" "$CURRENT_HEAD_SHA"
-      echo "${PR_BASE_REF}"
-      exit 0
-    fi
-  fi
-  if echo "$PR_DATA" | grep -q "^x-ratelimit-reset:"; then
-    reset_timestamp=$(echo -n "$PR_DATA" | grep "^x-ratelimit-reset:" | sed -e 's/^x-ratelimit-reset: //' -e 's/\r//')
-    now=$(date +%s)
-    sleep_time=$((reset_timestamp - now + 1))
-    echo "GitHub rate limit exceeded, sleeping for ${sleep_time} seconds" >&2
-    sleep "${sleep_time}"
-    continue
-  fi
-  echo -e "GitHub request failed for an unknown reason:\n$(echo "$PR_DATA" | sed '/^$/q')" >&2
-  exit 1
-done
+# If base ref is ambiguous, we cannot determine the correct one.
+# Example: a release branch is created, and a PR is opened starting from the
+# commit where the release branch was created. The distance to the merge base
+# for both master and the release branch is the same. In this case, we bail
+# out, and make no assumption on which is the correct base ref.
+echo "Base ref is ambiguous, candidates are: ${BEST_CANDIDATES[*]}" >&2
+exit 1

build.gradle

@@ -110,8 +110,8 @@ nexusPublishing {
       sonatype {
         nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/"))
         snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/"))
-        username = System.getenv("SONATYPE_USERNAME")
-        password = System.getenv("SONATYPE_PASSWORD")
+        username = System.getenv("MAVEN_CENTRAL_USERNAME")
+        password = System.getenv("MAVEN_CENTRAL_PASSWORD")
       }
     }
   }

communication/src/main/java/datadog/communication/BackendApiFactory.java

@@ -72,6 +72,7 @@ private HttpUrl getAgentlessUrl(Intake intake) {
 
   public enum Intake {
     API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl),
+    LLMOBS_API("api", "v2", Config::isLlmObsAgentlessEnabled, Config::getLlMObsAgentlessUrl),
     LOGS(
         "http-intake.logs",
         "v2",

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -30,7 +30,7 @@ public class SharedCommunicationObjects {
   public OkHttpClient okHttpClient;
   public HttpUrl agentUrl;
   public Monitoring monitoring;
-  private DDAgentFeaturesDiscovery featuresDiscovery;
+  private volatile DDAgentFeaturesDiscovery featuresDiscovery;
   private ConfigurationPoller configurationPoller;
 
   public SharedCommunicationObjects() {
@@ -139,28 +139,34 @@ public void setFeaturesDiscovery(DDAgentFeaturesDiscovery featuresDiscovery) {
   }
 
   public DDAgentFeaturesDiscovery featuresDiscovery(Config config) {
-    if (featuresDiscovery == null) {
-      createRemaining(config);
-      featuresDiscovery =
-          new DDAgentFeaturesDiscovery(
-              okHttpClient,
-              monitoring,
-              agentUrl,
-              config.isTraceAgentV05Enabled(),
-              config.isTracerMetricsEnabled());
-
-      if (paused) {
-        // defer remote discovery until remote I/O is allowed
-      } else {
-        if (AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) {
-          featuresDiscovery.discover(); // safe to run on same thread
-        } else {
-          // avoid performing blocking I/O operation on application thread
-          AgentTaskScheduler.INSTANCE.execute(featuresDiscovery::discover);
+    DDAgentFeaturesDiscovery ret = featuresDiscovery;
+    if (ret == null) {
+      synchronized (this) {
+        if (featuresDiscovery == null) {
+          createRemaining(config);
+          ret =
+              new DDAgentFeaturesDiscovery(
+                  okHttpClient,
+                  monitoring,
+                  agentUrl,
+                  config.isTraceAgentV05Enabled(),
+                  config.isTracerMetricsEnabled());
+
+          if (paused) {
+            // defer remote discovery until remote I/O is allowed
+          } else {
+            if (AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) {
+              ret.discover(); // safe to run on same thread
+            } else {
+              // avoid performing blocking I/O operation on application thread
+              AgentTaskScheduler.INSTANCE.execute(ret::discover);
+            }
+          }
+          featuresDiscovery = ret;
         }
       }
     }
-    return featuresDiscovery;
+    return ret;
   }
 
   private static final class FixedConfigUrlSupplier implements Supplier<String> {

components/environment/src/test/java/datadog/environment/CommandLineTest.java

@@ -7,7 +7,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import datadog.environment.CommandLineHelper.Result;
@@ -99,7 +99,7 @@ private static void skipArgFileTestOnJava8(RunArguments arguments) {
       }
     }
     if (useArgFile) {
-      assumeFalse(System.getProperty("java.home").matches(".*[-/]8[./].*"));
+      assumeTrue(JavaVirtualMachine.isJavaVersionAtLeast(9));
     }
   }
 

components/environment/src/test/java/datadog/environment/JavaVirtualMachineTest.java

@@ -114,15 +114,14 @@ void onlyOnGraalVm() {
   void onlyOnIbm8() {
     assertFalse(JavaVirtualMachine.isGraalVM());
     assertTrue(JavaVirtualMachine.isIbm8());
-    assertFalse(JavaVirtualMachine.isJ9());
+    assertTrue(JavaVirtualMachine.isJ9());
     assertFalse(JavaVirtualMachine.isOracleJDK8());
   }
 
   @Test
   @EnabledIfSystemProperty(named = "java.vm.name", matches = ".*J9.*")
   void onlyOnJ9() {
     assertFalse(JavaVirtualMachine.isGraalVM());
-    assertFalse(JavaVirtualMachine.isIbm8());
     assertTrue(JavaVirtualMachine.isJ9());
     assertFalse(JavaVirtualMachine.isOracleJDK8());
   }