Merge branch 'master' into jpbempel/update-probe-file-format

Author: jpbempel

.github/workflows/README.md

@@ -85,13 +85,6 @@ _Action:_
 
 _Recovery:_ Check at the milestone for the related issues and update them manually.
 
-### prune-github-container-registry [🔗](prune-github-container-registry.yaml)
-
-_Trigger:_ Every day or manually.
-
-_Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
-
-_Recovery:_ Manually trigger the action again.
 
 ### prune-old-pull-requests [🔗](prune-old-pull-requests.yaml)
 

.github/workflows/analyze-changes.yaml

@@ -40,7 +40,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -57,7 +57,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
 
   trivy:
     name: Analyze changes with Trivy
@@ -109,7 +109,7 @@ jobs:
           ls -laR "./workspace/.trivy"
 
       - name: Run Trivy security scanner
-        uses: aquasecurity/trivy-action@76071ef0d7ec797419534a183b498b4d6366cf37 # v0.31.0
+        uses: aquasecurity/trivy-action@dc5a429b52fcf669ce959baa2c2dd26090d2a6c4 # v0.32.0
         with:
           scan-type: rootfs
           scan-ref: './workspace/.trivy/'
@@ -122,7 +122,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/prune-github-container-registry.yaml

@@ -64,13 +64,15 @@ replay_pid*
 # Magic for local JMC built
 /vendor/jmc-libs
 
-# CircleCI #
-############
-_circle_ci_cache_*
-upstream.env
-/.circleci/config.continue.yml
-
 # Benchmarks #
 benchmark/reports
 benchmark/tracer
 benchmark/dacapo/scratch
+
+# JDK provisioning tools #
+# mise
+mise*.local.toml
+.mise*.local.toml
+.config/mise*.toml
+# asdf
+.tool-versions

.gitignore

@@ -41,9 +41,6 @@ variables:
     description: "Enable flaky tests"
     value: "false"
 
-default:
-  interruptible: true
-
 # trigger new commit cancel
 workflow:
   auto_cancel:
@@ -100,6 +97,7 @@ workflow:
 
 default:
   tags: [ "arch:amd64" ]
+  interruptible: true
 
 .set_datadog_api_keys: &set_datadog_api_keys
   - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
@@ -119,6 +117,18 @@ default:
   - .gitlab/cgroup-info.sh
   - gitlab_section_end "cgroup-info"
 
+.gitlab_base_ref_params: &gitlab_base_ref_params
+  - |
+    # FIXME: Disabled until we find a way to not hit GitHub API rate limit
+    if false && [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then
+      export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
+      if [[ -n "$GIT_BASE_REF" ]]; then
+        export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
+      else
+        echo "Failed to find base ref for PR" >&2
+      fi
+    fi
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
   stage: build
@@ -176,7 +186,32 @@ default:
   after_script:
     - *cgroup_info
 
+# Checks and fail early if central credentials are incorrect, indeed, when a new token is generated
+# on the central publisher protal, it invalidates the old one. This checks prevents going further.
+# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE
+pre-release-checks:
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
+  stage: .pre
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
+      when: on_success
+      allow_failure: false
+  script:
+    - |
+      SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
+      SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
+      # See https://central.sonatype.org/publish/publish-portal-api/
+      # 15e0cbbb-deff-421e-9e02-296a24d0cada is deployment, any deployment id listed in central  work, the idea is to check whether the token can authenticate
+      curl --request POST --include --fail https://central.sonatype.com/api/v1/publisher/status?id=15e0cbbb-deff-421e-9e02-296a24d0cada --header "Authorization: Bearer $(printf "$SONATYPE_USERNAME:$SONATYPE_PASSWORD" | base64)"
+      if [ $? -ne 0 ]; then
+        echo "Failed to authenticate against central. Check credentials, see https://datadoghq.atlassian.net/wiki/x/Oog5OgE"
+        exit 1
+      fi
+
 build:
+  needs:
+    - job: pre-release-checks
+      optional: true
   extends: .gradle_build
   variables:
     BUILD_CACHE_POLICY: push
@@ -223,7 +258,8 @@ build_tests:
         MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on
 
   script:
-    - ./gradlew clean $GRADLE_TARGET -PskipTests $GRADLE_ARGS
+    - *gitlab_base_ref_params
+    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS
 
 populate_dep_cache:
   extends: build_tests
@@ -313,7 +349,7 @@ test_published_artifacts:
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
-    - .circleci/collect_reports.sh
+    - .gitlab/collect_reports.sh
     - gitlab_section_end "collect-reports"
   artifacts:
     when: always
@@ -327,12 +363,13 @@ test_published_artifacts:
   variables:
     CACHE_TYPE: lib
   script:
-    - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+    - *gitlab_base_ref_params
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
   after_script:
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
-    - .circleci/collect_reports.sh --destination ./check_reports --move
+    - .gitlab/collect_reports.sh --destination ./check_reports --move
     - gitlab_section_end "collect-reports"
   artifacts:
     when: always
@@ -380,20 +417,19 @@ muzzle:
   extends: .gradle_build
   needs: [ build_tests ]
   stage: tests
-  parallel: 8
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"]
   variables:
     CACHE_TYPE: inst
   script:
     - export SKIP_BUILDSCAN="true"
-    - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
-    - sort workspace/build/muzzleTasks > sortedMuzzleTasks
-    - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
-    - ./gradlew $(cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs) $GRADLE_ARGS
+    - ./gradlew :runMuzzle -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
   after_script:
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
-    - .circleci/collect_reports.sh
+    - .gitlab/collect_reports.sh
     - gitlab_section_end "collect-reports"
   artifacts:
     when: always
@@ -412,7 +448,7 @@ muzzle-dep-report:
     - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
   after_script:
     - *cgroup_info
-    - .circleci/collect_muzzle_deps.sh
+    - .gitlab/collect_muzzle_deps.sh
   artifacts:
     when: always
     paths:
@@ -460,6 +496,7 @@ muzzle-dep-report:
     - if: $CI_COMMIT_BRANCH == "master"
       when: on_success
   script:
+    - *gitlab_base_ref_params
     - >
       if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
       then
@@ -474,10 +511,10 @@ muzzle-dep-report:
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
-    - .circleci/collect_reports.sh
-    - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
-    - .circleci/collect_results.sh
-    - .circleci/upload_ciapp.sh $CACHE_TYPE $testJvm
+    - .gitlab/collect_reports.sh
+    - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi
+    - .gitlab/collect_results.sh
+    - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
     - gitlab_section_end "collect-reports"
     - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
     - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
@@ -506,7 +543,7 @@ muzzle-dep-report:
     CI_USE_TEST_AGENT: "true"
     CI_AGENT_HOST: local-agent
   services:
-    - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+    - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.27.1
       alias: local-agent
       variables:
         LOG_LEVEL: "DEBUG"
@@ -729,6 +766,7 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
     UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
 
+# If the deploy_to_sonatype job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
 deploy_to_sonatype:
   extends: .gradle_build
   stage: publish
@@ -746,8 +784,8 @@ deploy_to_sonatype:
     - when: manual
       allow_failure: true
   script:
-    - export SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_username --with-decryption --query "Parameter.Value" --out text)
-    - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text)
+    - export SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
+    - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
     - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
@@ -765,7 +803,7 @@ deploy_artifacts_to_github:
       when: never
     - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
       when: on_success
-  # Requires the deploy_to_sonatype job to have run first the UP-TO-DATE gradle check across jobs is broken
+  # Requires the deploy_to_sonatype job to have run first (the UP-TO-DATE gradle check across jobs is broken)
   # This will deploy the artifacts built from the publishToSonatype task to the GitHub release
   needs:
     - job: deploy_to_sonatype

.gitlab-ci.yml

@@ -27,9 +27,6 @@
     UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
 
-    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-java
-    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
-
 benchmarks-startup:
   extends: .benchmarks
   script:
@@ -56,6 +53,7 @@ benchmarks-dacapo:
 
 benchmarks-post-results:
   extends: .benchmarks
+  tags: ["arch:amd64"]
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/upload-results-to-s3.sh
@@ -77,6 +75,10 @@ check-big-regressions:
       artifacts: true
   when: on_success
   tags: ["arch:amd64"]
+  rules:
+    - if: '$CI_COMMIT_BRANCH !~ /^(master|release\/)/'
+      when: on_success
+    - when: never
   # ARTIFACTS_DIR /go/src/github.com/DataDog/apm-reliability/dd-trace-java/reports/
   # need to convert them
   script:

.gitlab/benchmarks.yml

@@ -80,4 +80,3 @@ elif [ -d "/sys/fs/cgroup/memory" ]; then # Assuming if memory cgroup v1 exists,
 else
   printf "cgroup memory paths not found. Neither cgroup v2 controller file nor cgroup v1 memory directory detected.\n"
 fi
-

.gitlab/cgroup-info.sh

@@ -7,6 +7,10 @@ SUMMARY_RESPONSE_CODE=$(echo "$SUMMARY_RESPONSE" | awk 'END {print $NF}')
 
 if [[ SUMMARY_RESPONSE_CODE -eq 200 ]]; then
   echo "APM Test Agent is running. (HTTP 200)"
+elif [[ -n "$CI_USE_TEST_AGENT" ]]; then
+  echo "APM Test Agent failed to start, had an error, or exited early."
+  cat summary_response.txt
+  exit 1
 else
   echo "APM Test Agent is not running and was not used for testing. No checks failed."
   exit 0

.gitlab/check_test_agent_results.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Save all important profiles into (project-root)/profiles
-# This folder will be saved by circleci and available after test runs.
+# This folder will be saved by gitlab and available after test runs.
 
 set -e
 #Enable '**' support