From db53b07ef7681a83af13bfefc77529e4912ce13a Mon Sep 17 00:00:00 2001 From: Saranya Somepalli Date: Wed, 27 Aug 2025 10:28:33 -0700 Subject: [PATCH] Implement feature IDs: -S3Express, S3AccessGrants, IMDS Credentials --- ...sCredentialsBusinessMetricInterceptor.java | 51 ++ .../global/handlers/execution.interceptors | 1 + .../ImdsCredentialsInUserAgentTest.java | 120 ++++ .../useragent/BusinessMetricFeatureId.java | 5 +- .../s3/crt/CRTStuckStateMonitoringTest.java | 520 ++++++++++++++++++ .../services/s3/internal/BucketUtils.java | 11 + ...AccessGrantsBusinessMetricInterceptor.java | 90 +++ .../S3ExpressBusinessMetricInterceptor.java | 86 +++ .../awssdk/services/s3/execution.interceptors | 2 + .../s3/S3AccessGrantsInUserAgentTest.java | 241 ++++++++ .../services/s3/S3ExpressInUserAgentTest.java | 259 +++++++++ .../ssooidc/SsoLoginDeviceUserAgentTest.java | 96 ++++ 12 files changed, 1481 insertions(+), 1 deletion(-) create mode 100644 core/auth/src/main/java/software/amazon/awssdk/auth/credentials/internal/ImdsCredentialsBusinessMetricInterceptor.java create mode 100644 core/auth/src/main/resources/software/amazon/awssdk/global/handlers/execution.interceptors create mode 100644 core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ImdsCredentialsInUserAgentTest.java create mode 100644 services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/CRTStuckStateMonitoringTest.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3AccessGrantsBusinessMetricInterceptor.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3ExpressBusinessMetricInterceptor.java create mode 100644 services/s3/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/S3AccessGrantsInUserAgentTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/S3ExpressInUserAgentTest.java create mode 100644 services/ssooidc/src/test/java/software/amazon/awssdk/services/ssooidc/SsoLoginDeviceUserAgentTest.java diff --git a/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/internal/ImdsCredentialsBusinessMetricInterceptor.java b/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/internal/ImdsCredentialsBusinessMetricInterceptor.java new file mode 100644 index 000000000000..39245fef14a6 --- /dev/null +++ b/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/internal/ImdsCredentialsBusinessMetricInterceptor.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.auth.credentials.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; + +/** + * Interceptor that adds the CREDENTIALS_IMDS business metric when IMDS credentials are being used. + */ +@SdkInternalApi +public final class ImdsCredentialsBusinessMetricInterceptor implements ExecutionInterceptor { + + @Override + public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { + AwsCredentials credentials = executionAttributes.getAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS); + + if (credentials != null && isImdsCredentials(credentials)) { + executionAttributes.getAttribute(SdkInternalExecutionAttribute.BUSINESS_METRICS) + .addMetric(BusinessMetricFeatureId.CREDENTIALS_IMDS.value()); + } + + return context.request(); + } + + private boolean isImdsCredentials(AwsCredentials credentials) { + return credentials.providerName() + .map(name -> name.contains("InstanceProfile")) + .orElse(false); + } +} diff --git a/core/auth/src/main/resources/software/amazon/awssdk/global/handlers/execution.interceptors b/core/auth/src/main/resources/software/amazon/awssdk/global/handlers/execution.interceptors new file mode 100644 index 000000000000..166427e1f7a7 --- /dev/null +++ b/core/auth/src/main/resources/software/amazon/awssdk/global/handlers/execution.interceptors @@ -0,0 +1 @@ +software.amazon.awssdk.auth.credentials.internal.ImdsCredentialsBusinessMetricInterceptor diff --git a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ImdsCredentialsInUserAgentTest.java b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ImdsCredentialsInUserAgentTest.java new file mode 100644 index 000000000000..c7b93a2a2eee --- /dev/null +++ b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ImdsCredentialsInUserAgentTest.java @@ -0,0 +1,120 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.auth.credentials; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.internal.ImdsCredentialsBusinessMetricInterceptor; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.useragent.BusinessMetricCollection; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; + +public class ImdsCredentialsInUserAgentTest { + + private ImdsCredentialsBusinessMetricInterceptor interceptor; + private ExecutionAttributes executionAttributes; + private BusinessMetricCollection businessMetrics; + private Context.ModifyRequest context; + + @BeforeEach + void setUp() { + interceptor = new ImdsCredentialsBusinessMetricInterceptor(); + executionAttributes = new ExecutionAttributes(); + businessMetrics = new BusinessMetricCollection(); + executionAttributes.putAttribute(SdkInternalExecutionAttribute.BUSINESS_METRICS, businessMetrics); + + context = mock(Context.ModifyRequest.class); + SdkRequest request = mock(SdkRequest.class); + when(context.request()).thenReturn(request); + } + + @Test + public void imdsCredentials_shouldHaveImdsCredentialsBusinessMetric() { + // Create credentials with IMDS provider name + AwsCredentials imdsCredentials = createCredentialsWithProviderName("InstanceProfileCredentialsProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, imdsCredentials); + + interceptor.modifyRequest(context, executionAttributes); + + assertThat(businessMetrics.recordedMetrics()) + .contains(BusinessMetricFeatureId.CREDENTIALS_IMDS.value()); + } + + @Test + public void regularCredentials_shouldNotHaveImdsCredentialsBusinessMetric() { + // Create credentials with non-IMDS provider name + AwsCredentials regularCredentials = createCredentialsWithProviderName("DefaultCredentialsProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, regularCredentials); + + interceptor.modifyRequest(context, executionAttributes); + + assertThat(businessMetrics.recordedMetrics()) + .doesNotContain(BusinessMetricFeatureId.CREDENTIALS_IMDS.value()); + } + + @Test + public void containerCredentials_shouldNotHaveImdsCredentialsBusinessMetric() { + // Test with "ContainerCredentialsProvider" provider name - should NOT be considered IMDS + AwsCredentials containerCredentials = createCredentialsWithProviderName("ContainerCredentialsProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, containerCredentials); + + interceptor.modifyRequest(context, executionAttributes); + + assertThat(businessMetrics.recordedMetrics()) + .doesNotContain(BusinessMetricFeatureId.CREDENTIALS_IMDS.value()); + } + + @Test + public void credentialsWithoutProviderName_shouldNotHaveImdsCredentialsBusinessMetric() { + // Test with credentials that don't have a provider name + AwsCredentials credentialsWithoutProviderName = AwsBasicCredentials.create("accessKey", "secretKey"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, credentialsWithoutProviderName); + + interceptor.modifyRequest(context, executionAttributes); + + assertThat(businessMetrics.recordedMetrics()) + .doesNotContain(BusinessMetricFeatureId.CREDENTIALS_IMDS.value()); + } + + private AwsCredentials createCredentialsWithProviderName(String providerName) { + AwsCredentials baseCredentials = AwsBasicCredentials.create("accessKey", "secretKey"); + return new AwsCredentials() { + @Override + public String accessKeyId() { + return baseCredentials.accessKeyId(); + } + + @Override + public String secretAccessKey() { + return baseCredentials.secretAccessKey(); + } + + @Override + public Optional providerName() { + return Optional.of(providerName); + } + }; + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java index 7f1483d56895..bc7936cfd37c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java @@ -22,7 +22,7 @@ /** * An enum class representing a short form of identity providers to record in the UA string. * - * Unimplemented metrics: I,J,K,M,O,S,U-c,e-[latest] + * Unimplemented metrics: I,M,O,S,U-c,e-[latest] * Unsupported metrics (these will never be added): A,H */ @SdkProtectedApi @@ -34,6 +34,8 @@ public enum BusinessMetricFeatureId { RETRY_MODE_STANDARD("E"), RETRY_MODE_ADAPTIVE("F"), S3_TRANSFER("G"), + S3_EXPRESS_BUCKET("J"), + S3_ACCESS_GRANTS("K"), GZIP_REQUEST_COMPRESSION("L"), //TODO(metrics): Not working, compression happens after header ENDPOINT_OVERRIDE("N"), ACCOUNT_ID_MODE_PREFERRED("P"), @@ -42,6 +44,7 @@ public enum BusinessMetricFeatureId { RESOLVED_ACCOUNT_ID("T"), DDB_MAPPER("d"), BEARER_SERVICE_ENV_VARS("3"), + CREDENTIALS_IMDS("0"), UNKNOWN("Unknown"); private static final Map VALUE_MAP = diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/CRTStuckStateMonitoringTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/CRTStuckStateMonitoringTest.java new file mode 100644 index 000000000000..7c8e6bed60f0 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/CRTStuckStateMonitoringTest.java @@ -0,0 +1,520 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crt; + +import static org.assertj.core.api.Assertions.*; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +/** + * Advanced integration test that monitors system resources to detect the "stuck state" issue + * in S3 CRT client after BlockingInputStreamAsyncRequestBody timeouts. + * + * This test combines: + * 1. Reproduction of the timeout issue + * 2. Real-time system monitoring (CPU, memory, threads) + * 3. Detection of stuck state indicators + * 4. Validation of client recovery capabilities + */ +public class CRTStuckStateMonitoringTest extends S3IntegrationTestBase { + + private S3AsyncClient s3CrtClient; + private ScheduledExecutorService monitoringExecutor; + private SystemMonitor systemMonitor; + protected String bucketName; + + // Monitoring thresholds + private static final double HIGH_CPU_THRESHOLD = 80.0; // 80% CPU usage + private static final long HIGH_MEMORY_THRESHOLD = 500 * 1024 * 1024; // 500MB + private static final int HIGH_THREAD_THRESHOLD = 50; // 50+ threads + private static final Duration MONITORING_INTERVAL = Duration.ofSeconds(2); + private static final Duration TEST_DURATION = Duration.ofSeconds(60); + + @BeforeEach + void setUpTest() { + // Initialize base S3 client if needed + if (s3 == null) { + try { + setUp(); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize S3 client", e); + } + } + + // Create S3 CRT Async Client + s3CrtClient = S3AsyncClient.crtBuilder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + + // Initialize monitoring + monitoringExecutor = Executors.newScheduledThreadPool(2); + systemMonitor = new SystemMonitor(); + + // Create test bucket + bucketName = "crt-stuck-state-monitor-" + System.currentTimeMillis(); + createBucket(bucketName); + + System.out.println("=== CRT Stuck State Monitoring Test Started ==="); + System.out.println("Bucket: " + bucketName); + System.out.println("Monitoring thresholds - CPU: " + HIGH_CPU_THRESHOLD + "%, Memory: " + + (HIGH_MEMORY_THRESHOLD / 1024 / 1024) + "MB, Threads: " + HIGH_THREAD_THRESHOLD); + } + + @AfterEach + void tearDownTest() { + // Stop monitoring first + if (monitoringExecutor != null) { + monitoringExecutor.shutdown(); + try { + monitoringExecutor.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + // Close CRT client + if (s3CrtClient != null) { + try { + s3CrtClient.close(); + } catch (Exception e) { + System.err.println("Failed to close S3 CRT client: " + e.getMessage()); + } + } + + // Clean up bucket + if (bucketName != null) { + try { + deleteBucketAndAllContents(bucketName); + } catch (Exception e) { + System.err.println("Failed to delete test bucket: " + e.getMessage()); + } + } + + // Print final monitoring report + if (systemMonitor != null) { + systemMonitor.printFinalReport(); + } + } + + /** + * Test that reproduces the stuck state issue and monitors system resources + * to detect the characteristic patterns of the bug. + */ + @Test + @Timeout(120) + void testStuckStateWithSystemMonitoring() throws Exception { + System.out.println("--- Starting Stuck State Detection Test ---"); + + // Start system monitoring + systemMonitor.startMonitoring(monitoringExecutor); + + // Phase 1: Trigger the timeout issue + System.out.println("Phase 1: Triggering timeout to cause stuck state..."); + boolean timeoutOccurred = triggerTimeoutIssue(); + + assertThat(timeoutOccurred).isTrue(); + System.out.println("✓ Timeout successfully triggered"); + + // Phase 2: Monitor for stuck state indicators + System.out.println("Phase 2: Monitoring for stuck state indicators..."); + Thread.sleep(10000); // Wait 10 seconds for stuck state to manifest + + StuckStateIndicators indicators = systemMonitor.analyzeStuckState(); + + // Phase 3: Test client recovery + System.out.println("Phase 3: Testing client recovery..."); + boolean recoverySuccessful = testClientRecovery(); + + // Phase 4: Analysis and reporting + System.out.println("--- Analysis Results ---"); + + if (indicators.isStuckStateDetected()) { + System.out.println("✗ STUCK STATE DETECTED!"); + System.out.println(" - High CPU: " + indicators.hasHighCpu()); + System.out.println(" - Memory Issues: " + indicators.hasMemoryIssues()); + System.out.println(" - Thread Problems: " + indicators.hasThreadIssues()); + System.out.println(" - CRT Threads Stuck: " + indicators.hasCrtThreadsStuck()); + + if (!recoverySuccessful) { + fail("S3 CRT Client entered stuck state after timeout. " + + "Client recovery failed. Indicators: " + indicators); + } + } else { + System.out.println("✓ No stuck state detected - client appears healthy"); + } + + if (recoverySuccessful) { + System.out.println("✓ Client recovery successful"); + } else { + System.out.println("✗ Client recovery failed"); + } + } + + /** + * Test that reproduces the exact user pattern that causes the issue + */ + @Test + @Timeout(90) + void testUserPatternReproduction() throws Exception { + System.out.println("--- Testing User Pattern Reproduction ---"); + + systemMonitor.startMonitoring(monitoringExecutor); + + // Reproduce the exact pattern from user code analysis + byte[] testData = "User pattern test data - this simulates the CloudWatch Elephant service pattern".getBytes(); + + try { + // Step 1: Create request body with null content length (user's pattern) + BlockingInputStreamAsyncRequestBody requestBody = + BlockingInputStreamAsyncRequestBody.builder() + .contentLength(null) // User passes null + .subscribeTimeout(Duration.ofSeconds(5)) // Shorter timeout to trigger issue + .build(); + + // Step 2: Start S3 operation (user's pattern) + PutObjectRequest putRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key("user-pattern-test-key") + .build(); + + CompletableFuture putFuture = s3CrtClient.putObject(putRequest, requestBody); + + // Step 3: Immediately try to write input stream (user's pattern) + // This creates the race condition that leads to timeout + InputStream inputStream = new ByteArrayInputStream(testData); + + long startTime = System.currentTimeMillis(); + requestBody.writeInputStream(inputStream); + + // If we get here without timeout, wait for completion + putFuture.get(30, TimeUnit.SECONDS); + + System.out.println("✓ User pattern succeeded (no timeout occurred)"); + + } catch (IllegalStateException e) { + long duration = System.currentTimeMillis() - startTime; + System.out.println("✓ User pattern reproduced timeout after " + duration + "ms"); + System.out.println(" Exception: " + e.getMessage()); + + // Now monitor for stuck state + Thread.sleep(5000); + StuckStateIndicators indicators = systemMonitor.analyzeStuckState(); + + if (indicators.isStuckStateDetected()) { + System.out.println("✗ CRITICAL: User pattern caused stuck state!"); + System.out.println(" This confirms the production issue reported in the ticket"); + } + } + } + + /** + * Test concurrent operations to simulate production load + */ + @Test + @Timeout(150) + void testConcurrentOperationsStuckState() throws Exception { + System.out.println("--- Testing Concurrent Operations for Stuck State ---"); + + systemMonitor.startMonitoring(monitoringExecutor); + + int concurrentOperations = 5; + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completionLatch = new CountDownLatch(concurrentOperations); + + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger timeoutCount = new AtomicInteger(0); + AtomicInteger errorCount = new AtomicInteger(0); + + // Start concurrent operations + for (int i = 0; i < concurrentOperations; i++) { + final int operationId = i; + + CompletableFuture.runAsync(() -> { + try { + startLatch.await(); + + byte[] data = ("Concurrent operation " + operationId + " data").getBytes(); + + BlockingInputStreamAsyncRequestBody requestBody = + BlockingInputStreamAsyncRequestBody.builder() + .contentLength((long) data.length) + .subscribeTimeout(Duration.ofSeconds(3)) // Short timeout + .build(); + + PutObjectRequest putRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key("concurrent-key-" + operationId) + .build(); + + CompletableFuture putFuture = s3CrtClient.putObject(putRequest, requestBody); + requestBody.writeInputStream(new ByteArrayInputStream(data)); + putFuture.get(30, TimeUnit.SECONDS); + + successCount.incrementAndGet(); + System.out.println("✓ Concurrent operation " + operationId + " succeeded"); + + } catch (IllegalStateException e) { + timeoutCount.incrementAndGet(); + System.out.println("⚠ Concurrent operation " + operationId + " timed out"); + + } catch (Exception e) { + errorCount.incrementAndGet(); + System.out.println("✗ Concurrent operation " + operationId + " failed: " + e.getMessage()); + + } finally { + completionLatch.countDown(); + } + }); + } + + // Start all operations simultaneously + startLatch.countDown(); + + // Wait for completion + completionLatch.await(60, TimeUnit.SECONDS); + + System.out.println("--- Concurrent Operations Results ---"); + System.out.println("Successes: " + successCount.get()); + System.out.println("Timeouts: " + timeoutCount.get()); + System.out.println("Errors: " + errorCount.get()); + + // Analyze stuck state after concurrent operations + Thread.sleep(5000); + StuckStateIndicators indicators = systemMonitor.analyzeStuckState(); + + if (indicators.isStuckStateDetected()) { + System.out.println("✗ CRITICAL: Concurrent operations caused stuck state!"); + System.out.println(" Success rate: " + successCount.get() + "/" + concurrentOperations); + + if (successCount.get() == 0 && timeoutCount.get() > 0) { + fail("All concurrent operations failed after initial timeout - indicates stuck state"); + } + } + } + + private boolean triggerTimeoutIssue() { + try { + byte[] testData = "Timeout trigger test data".getBytes(); + + BlockingInputStreamAsyncRequestBody requestBody = + BlockingInputStreamAsyncRequestBody.builder() + .contentLength((long) testData.length) + .subscribeTimeout(Duration.ofSeconds(1)) // Very short timeout + .build(); + + // Call writeInputStream BEFORE putObject to guarantee timeout + InputStream inputStream = new ByteArrayInputStream(testData); + requestBody.writeInputStream(inputStream); + + return false; // If we get here, no timeout occurred + + } catch (IllegalStateException e) { + return e.getMessage().contains("service request was not made within"); + } catch (Exception e) { + System.err.println("Unexpected exception during timeout trigger: " + e.getMessage()); + return false; + } + } + + private boolean testClientRecovery() { + try { + byte[] testData = "Recovery test data".getBytes(); + + BlockingInputStreamAsyncRequestBody requestBody = + AsyncRequestBody.forBlockingInputStream((long) testData.length); + + PutObjectRequest putRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key("recovery-test-key") + .build(); + + CompletableFuture putFuture = s3CrtClient.putObject(putRequest, requestBody); + requestBody.writeInputStream(new ByteArrayInputStream(testData)); + putFuture.get(30, TimeUnit.SECONDS); + + return true; + + } catch (Exception e) { + System.err.println("Client recovery failed: " + e.getMessage()); + return false; + } + } + + /** + * System monitor that tracks CPU, memory, threads, and CRT-specific indicators + */ + private static class SystemMonitor { + private final OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + private final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + private final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + private final AtomicLong maxCpuUsage = new AtomicLong(0); + private final AtomicLong maxMemoryUsage = new AtomicLong(0); + private final AtomicInteger maxThreadCount = new AtomicInteger(0); + private final AtomicInteger highCpuCount = new AtomicInteger(0); + private final AtomicBoolean crtThreadsStuck = new AtomicBoolean(false); + + private final List stuckThreads = new ArrayList<>(); + + void startMonitoring(ScheduledExecutorService executor) { + executor.scheduleAtFixedRate(this::collectMetrics, 0, + MONITORING_INTERVAL.toMillis(), TimeUnit.MILLISECONDS); + } + + private void collectMetrics() { + // CPU monitoring + double cpuUsage = osBean.getProcessCpuLoad() * 100; + if (cpuUsage > 0) { // -1 indicates not available + maxCpuUsage.updateAndGet(current -> Math.max(current, (long) cpuUsage)); + + if (cpuUsage > HIGH_CPU_THRESHOLD) { + highCpuCount.incrementAndGet(); + checkForStuckThreads(); + } + } + + // Memory monitoring + long usedMemory = memoryBean.getHeapMemoryUsage().getUsed(); + maxMemoryUsage.updateAndGet(current -> Math.max(current, usedMemory)); + + // Thread monitoring + int threadCount = threadBean.getThreadCount(); + maxThreadCount.updateAndGet(current -> Math.max(current, threadCount)); + + // Log current state + System.out.printf("[MONITOR] CPU: %.1f%%, Memory: %dMB, Threads: %d%n", + cpuUsage, usedMemory / (1024 * 1024), threadCount); + } + + private void checkForStuckThreads() { + ThreadInfo[] threads = threadBean.dumpAllThreads(false, false); + + for (ThreadInfo thread : threads) { + String threadName = thread.getThreadName(); + + // Look for CRT-related threads + if (threadName.contains("aws-crt") || + threadName.contains("crt") || + threadName.contains("s3-meta-request")) { + + Thread.State state = thread.getThreadState(); + + // Check for problematic states + if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) { + String threadInfo = threadName + " (" + state + ")"; + + if (!stuckThreads.contains(threadInfo)) { + stuckThreads.add(threadInfo); + crtThreadsStuck.set(true); + System.out.println("[MONITOR] Potentially stuck CRT thread: " + threadInfo); + } + } + } + } + } + + StuckStateIndicators analyzeStuckState() { + return new StuckStateIndicators( + maxCpuUsage.get() > HIGH_CPU_THRESHOLD, + maxMemoryUsage.get() > HIGH_MEMORY_THRESHOLD, + maxThreadCount.get() > HIGH_THREAD_THRESHOLD, + crtThreadsStuck.get(), + highCpuCount.get() > 3 // High CPU for multiple monitoring cycles + ); + } + + void printFinalReport() { + System.out.println("=== Final System Monitoring Report ==="); + System.out.println("Max CPU Usage: " + maxCpuUsage.get() + "%"); + System.out.println("Max Memory Usage: " + (maxMemoryUsage.get() / 1024 / 1024) + "MB"); + System.out.println("Max Thread Count: " + maxThreadCount.get()); + System.out.println("High CPU Cycles: " + highCpuCount.get()); + System.out.println("CRT Threads Stuck: " + crtThreadsStuck.get()); + + if (!stuckThreads.isEmpty()) { + System.out.println("Stuck Threads Detected:"); + stuckThreads.forEach(thread -> System.out.println(" - " + thread)); + } + } + } + + /** + * Data class to hold stuck state analysis results + */ + private static class StuckStateIndicators { + private final boolean highCpu; + private final boolean memoryIssues; + private final boolean threadIssues; + private final boolean crtThreadsStuck; + private final boolean persistentHighCpu; + + StuckStateIndicators(boolean highCpu, boolean memoryIssues, boolean threadIssues, + boolean crtThreadsStuck, boolean persistentHighCpu) { + this.highCpu = highCpu; + this.memoryIssues = memoryIssues; + this.threadIssues = threadIssues; + this.crtThreadsStuck = crtThreadsStuck; + this.persistentHighCpu = persistentHighCpu; + } + + boolean isStuckStateDetected() { + // Stuck state is indicated by multiple symptoms + return (highCpu && crtThreadsStuck) || + (persistentHighCpu && threadIssues) || + (crtThreadsStuck && memoryIssues); + } + + boolean hasHighCpu() { return highCpu; } + boolean hasMemoryIssues() { return memoryIssues; } + boolean hasThreadIssues() { return threadIssues; } + boolean hasCrtThreadsStuck() { return crtThreadsStuck; } + + @Override + public String toString() { + return String.format("StuckStateIndicators{highCpu=%s, memoryIssues=%s, threadIssues=%s, " + + "crtThreadsStuck=%s, persistentHighCpu=%s}", + highCpu, memoryIssues, threadIssues, crtThreadsStuck, persistentHighCpu); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/BucketUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/BucketUtils.java index 9e6a31a7531d..5106446b92c0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/BucketUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/BucketUtils.java @@ -156,6 +156,17 @@ public static boolean isVirtualAddressingCompatibleBucketName(final String bucke return isValidDnsBucketName(bucketName, throwOnError) && !bucketName.contains("."); } + /** + * Determines if the given bucket name is an S3 Express bucket. + * S3 Express buckets follow the naming pattern: bucketname--azid--x-s3 + * + * @param bucketName The bucket name to check. + * @return true if the bucket name is an S3 Express bucket, false otherwise. + */ + public static boolean isS3ExpressBucket(String bucketName) { + return bucketName != null && bucketName.endsWith("--x-s3"); + } + /** * If 'exception' is true, throw an IllegalArgumentException with the given * message. Otherwise, silently return false. diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3AccessGrantsBusinessMetricInterceptor.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3AccessGrantsBusinessMetricInterceptor.java new file mode 100644 index 000000000000..03ec1c6bd02b --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3AccessGrantsBusinessMetricInterceptor.java @@ -0,0 +1,90 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.handlers; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; +import software.amazon.awssdk.services.s3.model.S3Request; + +/** + * Interceptor that adds business metrics for S3 Access Grants operations. + * This interceptor detects when an operation is performed using S3 Access Grants + * credentials and adds the appropriate business metric to track usage. + */ + +@SdkInternalApi +public final class S3AccessGrantsBusinessMetricInterceptor implements ExecutionInterceptor { + + private static final ApiName S3_ACCESS_GRANTS_API_NAME = ApiName.builder() + .name("sdk-metrics") + .version(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value()) + .build(); + + private static final Consumer S3_ACCESS_GRANTS_USER_AGENT_APPLIER = + b -> b.addApiName(S3_ACCESS_GRANTS_API_NAME); + + @Override + public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { + SdkRequest request = context.request(); + + if (request instanceof S3Request) { + S3Request s3Request = (S3Request) request; + + if (isUsingS3AccessGrantsCredentials(executionAttributes)) { + + AwsRequestOverrideConfiguration overrideConfiguration = + s3Request.overrideConfiguration() + .map(c -> c.toBuilder() + .applyMutation(S3_ACCESS_GRANTS_USER_AGENT_APPLIER) + .build()) + .orElseGet(() -> AwsRequestOverrideConfiguration.builder() + .applyMutation(S3_ACCESS_GRANTS_USER_AGENT_APPLIER) + .build()); + + return s3Request.toBuilder().overrideConfiguration(overrideConfiguration).build(); + } + } + + return request; + } + + /** + * Determines if S3 Access Grants credentials are being used for this request. + */ + private boolean isUsingS3AccessGrantsCredentials(ExecutionAttributes executionAttributes) { + AwsCredentials credentials = executionAttributes.getAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS); + + if (credentials == null) { + return false; + } + + // Check if the credentials have a provider name indicating they came from S3 Access Grants + return credentials.providerName() + .map(name -> name.contains("S3AccessGrants") || + name.contains("AccessGrants") || + name.contains("S3AccessGrantsIdentityProvider")) + .orElse(false); + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3ExpressBusinessMetricInterceptor.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3ExpressBusinessMetricInterceptor.java new file mode 100644 index 000000000000..b5cd7521cce0 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/S3ExpressBusinessMetricInterceptor.java @@ -0,0 +1,86 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.handlers; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; +import software.amazon.awssdk.services.s3.internal.BucketUtils; +import software.amazon.awssdk.services.s3.internal.s3express.S3ExpressUtils; +import software.amazon.awssdk.services.s3.model.S3Request; + +/** + * Interceptor that adds business metrics for S3 Express bucket operations. + * This interceptor detects when an operation is performed on an S3 Express bucket + * using S3 Express credentials and adds the appropriate business metric to track usage. + */ + +@SdkInternalApi +public final class S3ExpressBusinessMetricInterceptor implements ExecutionInterceptor { + + private static final ApiName S3_EXPRESS_API_NAME = ApiName.builder() + .name("sdk-metrics") + .version(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value()) + .build(); + + private static final Consumer S3_EXPRESS_USER_AGENT_APPLIER = + b -> b.addApiName(S3_EXPRESS_API_NAME); + + @Override + public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { + SdkRequest request = context.request(); + + if (request instanceof S3Request) { + S3Request s3Request = (S3Request) request; + String bucketName = getBucketName(s3Request); + + if (bucketName != null && + BucketUtils.isS3ExpressBucket(bucketName) && + S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) { + + AwsRequestOverrideConfiguration overrideConfiguration = + s3Request.overrideConfiguration() + .map(c -> c.toBuilder() + .applyMutation(S3_EXPRESS_USER_AGENT_APPLIER) + .build()) + .orElseGet(() -> AwsRequestOverrideConfiguration.builder() + .applyMutation(S3_EXPRESS_USER_AGENT_APPLIER) + .build()); + + return s3Request.toBuilder().overrideConfiguration(overrideConfiguration).build(); + } + } + + return request; + } + + /** + * Extracts the bucket name from an S3 request. + */ + private String getBucketName(S3Request request) { + try { + return (String) request.getClass().getMethod("bucket").invoke(request); + } catch (Exception e) { + return null; + } + } +} diff --git a/services/s3/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors b/services/s3/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors new file mode 100644 index 000000000000..404d81616e13 --- /dev/null +++ b/services/s3/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors @@ -0,0 +1,2 @@ +software.amazon.awssdk.services.s3.internal.handlers.S3AccessGrantsBusinessMetricInterceptor +software.amazon.awssdk.services.s3.internal.handlers.S3ExpressBusinessMetricInterceptor diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3AccessGrantsInUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3AccessGrantsInUserAgentTest.java new file mode 100644 index 000000000000..f7e390503657 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3AccessGrantsInUserAgentTest.java @@ -0,0 +1,241 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; +import software.amazon.awssdk.services.s3.internal.handlers.S3AccessGrantsBusinessMetricInterceptor; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Request; + +public class S3AccessGrantsInUserAgentTest { + + private S3AccessGrantsBusinessMetricInterceptor interceptor; + private ExecutionAttributes executionAttributes; + private Context.ModifyRequest context; + + @BeforeEach + void setUp() { + interceptor = new S3AccessGrantsBusinessMetricInterceptor(); + executionAttributes = new ExecutionAttributes(); + + context = mock(Context.ModifyRequest.class); + } + + @Test + public void s3AccessGrantsCredentials_shouldHaveS3AccessGrantsApiName() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Create credentials with S3 Access Grants provider name + AwsCredentials s3AccessGrantsCredentials = createCredentialsWithProviderName("S3AccessGrantsIdentityProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, s3AccessGrantsCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has been modified with S3 Access Grants API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + assertThat(modifiedS3Request.overrideConfiguration()).isPresent(); + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + + // Check that the API name contains the S3 Access Grants feature ID + assertThat(overrideConfig.apiNames()).isNotEmpty(); + boolean hasS3AccessGrantsApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value())); + assertThat(hasS3AccessGrantsApiName).isTrue(); + } + + @Test + public void regularCredentials_shouldNotHaveS3AccessGrantsApiName() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Create credentials with regular provider name + AwsCredentials regularCredentials = createCredentialsWithProviderName("DefaultCredentialsProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, regularCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified with S3 Access Grants API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + // Either no override configuration or no S3 Access Grants API name + if (modifiedS3Request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + boolean hasS3AccessGrantsApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value())); + assertThat(hasS3AccessGrantsApiName).isFalse(); + } + } + + @Test + public void accessGrantsProviderName_shouldHaveS3AccessGrantsApiName() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Test with "AccessGrants" provider name variation + AwsCredentials accessGrantsCredentials = createCredentialsWithProviderName("AccessGrants"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, accessGrantsCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has been modified with S3 Access Grants API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + assertThat(modifiedS3Request.overrideConfiguration()).isPresent(); + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + + boolean hasS3AccessGrantsApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value())); + assertThat(hasS3AccessGrantsApiName).isTrue(); + } + + @Test + public void s3AccessGrantsProviderName_shouldHaveS3AccessGrantsApiName() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Test with "S3AccessGrants" provider name variation + AwsCredentials s3AccessGrantsCredentials = createCredentialsWithProviderName("S3AccessGrants"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, s3AccessGrantsCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has been modified with S3 Access Grants API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + assertThat(modifiedS3Request.overrideConfiguration()).isPresent(); + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + + boolean hasS3AccessGrantsApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value())); + assertThat(hasS3AccessGrantsApiName).isTrue(); + } + + @Test + public void invalidProviderName_shouldNotHaveS3AccessGrantsApiName() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Test with provider name that should NOT trigger S3 Access Grants metric + AwsCredentials invalidCredentials = createCredentialsWithProviderName("InstanceProfileCredentialsProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, invalidCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified with S3 Access Grants API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + // Either no override configuration or no S3 Access Grants API name + if (modifiedS3Request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + boolean hasS3AccessGrantsApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_ACCESS_GRANTS.value())); + assertThat(hasS3AccessGrantsApiName).isFalse(); + } + } + + @Test + public void nonS3Request_shouldNotBeModified() { + // Create non-S3 request + SdkRequest nonS3Request = mock(SdkRequest.class); + when(context.request()).thenReturn(nonS3Request); + + // Create credentials with S3 Access Grants provider name + AwsCredentials s3AccessGrantsCredentials = createCredentialsWithProviderName("S3AccessGrantsIdentityProvider"); + executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, s3AccessGrantsCredentials); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified (should return the same request) + assertThat(modifiedRequest).isSameAs(nonS3Request); + } + + @Test + public void noCredentials_shouldNotModifyRequest() { + // Create S3 request + S3Request s3Request = PutObjectRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // No credentials set + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified (should return the same request) + assertThat(modifiedRequest).isSameAs(s3Request); + } + + private AwsCredentials createCredentialsWithProviderName(String providerName) { + AwsCredentials baseCredentials = AwsBasicCredentials.create("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + return new AwsCredentials() { + @Override + public String accessKeyId() { + return baseCredentials.accessKeyId(); + } + + @Override + public String secretAccessKey() { + return baseCredentials.secretAccessKey(); + } + + @Override + public Optional providerName() { + return Optional.of(providerName); + } + }; + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3ExpressInUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3ExpressInUserAgentTest.java new file mode 100644 index 000000000000..2f2629e6ad78 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3ExpressInUserAgentTest.java @@ -0,0 +1,259 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; +import software.amazon.awssdk.services.s3.internal.BucketUtils; +import software.amazon.awssdk.services.s3.internal.handlers.S3ExpressBusinessMetricInterceptor; +import software.amazon.awssdk.services.s3.internal.s3express.S3ExpressUtils; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Request; + +public class S3ExpressInUserAgentTest { + + private S3ExpressBusinessMetricInterceptor interceptor; + private ExecutionAttributes executionAttributes; + private Context.ModifyRequest context; + + @BeforeEach + void setUp() { + interceptor = new S3ExpressBusinessMetricInterceptor(); + executionAttributes = new ExecutionAttributes(); + + context = mock(Context.ModifyRequest.class); + } + + @Test + public void s3ExpressBucketWithS3ExpressAuth_shouldHaveS3ExpressApiName() { + // Create S3 request with S3 Express bucket name + S3Request s3Request = PutObjectRequest.builder() + .bucket("my-bucket--x-s3") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Mock BucketUtils and S3ExpressUtils + try (MockedStatic mockedBucketUtils = mockStatic(BucketUtils.class); + MockedStatic mockedS3ExpressUtils = mockStatic(S3ExpressUtils.class)) { + + mockedBucketUtils.when(() -> BucketUtils.isS3ExpressBucket("my-bucket--x-s3")) + .thenReturn(true); + mockedS3ExpressUtils.when(() -> S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) + .thenReturn(true); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has been modified with S3 Express API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + assertThat(modifiedS3Request.overrideConfiguration()).isPresent(); + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + + // Check that the API name contains the S3 Express feature ID + assertThat(overrideConfig.apiNames()).isNotEmpty(); + boolean hasS3ExpressApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value())); + assertThat(hasS3ExpressApiName).isTrue(); + } + } + + @Test + public void regularBucket_shouldNotHaveS3ExpressApiName() { + // Create S3 request with regular bucket name + S3Request s3Request = PutObjectRequest.builder() + .bucket("regular-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Mock BucketUtils and S3ExpressUtils + try (MockedStatic mockedBucketUtils = mockStatic(BucketUtils.class); + MockedStatic mockedS3ExpressUtils = mockStatic(S3ExpressUtils.class)) { + + mockedBucketUtils.when(() -> BucketUtils.isS3ExpressBucket("regular-bucket")) + .thenReturn(false); + mockedS3ExpressUtils.when(() -> S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) + .thenReturn(false); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified with S3 Express API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + // Either no override configuration or no S3 Express API name + if (modifiedS3Request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + boolean hasS3ExpressApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value())); + assertThat(hasS3ExpressApiName).isFalse(); + } + } + } + + @Test + public void s3ExpressBucketWithRegularAuth_shouldNotHaveS3ExpressApiName() { + // Create S3 request with S3 Express bucket name but regular auth + S3Request s3Request = PutObjectRequest.builder() + .bucket("my-bucket--x-s3") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Mock BucketUtils and S3ExpressUtils + try (MockedStatic mockedBucketUtils = mockStatic(BucketUtils.class); + MockedStatic mockedS3ExpressUtils = mockStatic(S3ExpressUtils.class)) { + + mockedBucketUtils.when(() -> BucketUtils.isS3ExpressBucket("my-bucket--x-s3")) + .thenReturn(true); + mockedS3ExpressUtils.when(() -> S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) + .thenReturn(false); // Using regular SigV4, not S3 Express auth + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified with S3 Express API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + // Either no override configuration or no S3 Express API name + if (modifiedS3Request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + boolean hasS3ExpressApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value())); + assertThat(hasS3ExpressApiName).isFalse(); + } + } + } + + @Test + public void regularBucketWithS3ExpressAuth_shouldNotHaveS3ExpressApiName() { + // Create S3 request with regular bucket name but S3 Express auth (edge case) + S3Request s3Request = PutObjectRequest.builder() + .bucket("regular-bucket") + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + // Mock BucketUtils and S3ExpressUtils + try (MockedStatic mockedBucketUtils = mockStatic(BucketUtils.class); + MockedStatic mockedS3ExpressUtils = mockStatic(S3ExpressUtils.class)) { + + mockedBucketUtils.when(() -> BucketUtils.isS3ExpressBucket("regular-bucket")) + .thenReturn(false); + mockedS3ExpressUtils.when(() -> S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) + .thenReturn(true); // Using S3 Express auth but regular bucket + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified with S3 Express API name + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + // Either no override configuration or no S3 Express API name + if (modifiedS3Request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + boolean hasS3ExpressApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value())); + assertThat(hasS3ExpressApiName).isFalse(); + } + } + } + + @Test + public void nonS3Request_shouldNotBeModified() { + // Create non-S3 request + SdkRequest nonS3Request = mock(SdkRequest.class); + when(context.request()).thenReturn(nonS3Request); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified (should return the same request) + assertThat(modifiedRequest).isSameAs(nonS3Request); + } + + @Test + public void s3RequestWithNullBucket_shouldNotBeModified() { + // Create S3 request with null bucket (edge case) + S3Request s3Request = PutObjectRequest.builder() + .key("test-key") + .build(); + when(context.request()).thenReturn(s3Request); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has NOT been modified (should return the same request) + assertThat(modifiedRequest).isSameAs(s3Request); + } + + @Test + public void s3ExpressBucketWithExistingOverrideConfig_shouldPreserveExistingConfig() { + // Create S3 request with existing override configuration + ApiName existingApiName = ApiName.builder().name("existing").version("1.0").build(); + AwsRequestOverrideConfiguration existingConfig = AwsRequestOverrideConfiguration.builder() + .addApiName(existingApiName) + .build(); + + S3Request s3Request = PutObjectRequest.builder() + .bucket("my-bucket--x-s3") + .key("test-key") + .overrideConfiguration(existingConfig) + .build(); + when(context.request()).thenReturn(s3Request); + + // Mock BucketUtils and S3ExpressUtils + try (MockedStatic mockedBucketUtils = mockStatic(BucketUtils.class); + MockedStatic mockedS3ExpressUtils = mockStatic(S3ExpressUtils.class)) { + + mockedBucketUtils.when(() -> BucketUtils.isS3ExpressBucket("my-bucket--x-s3")) + .thenReturn(true); + mockedS3ExpressUtils.when(() -> S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes)) + .thenReturn(true); + + SdkRequest modifiedRequest = interceptor.modifyRequest(context, executionAttributes); + + // Verify that the request has been modified with S3 Express API name AND preserves existing config + assertThat(modifiedRequest).isInstanceOf(S3Request.class); + S3Request modifiedS3Request = (S3Request) modifiedRequest; + + assertThat(modifiedS3Request.overrideConfiguration()).isPresent(); + AwsRequestOverrideConfiguration overrideConfig = modifiedS3Request.overrideConfiguration().get(); + + // Check that both existing and S3 Express API names are present + assertThat(overrideConfig.apiNames()).hasSize(2); + boolean hasExistingApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.name().equals("existing") && apiName.version().equals("1.0")); + boolean hasS3ExpressApiName = overrideConfig.apiNames().stream() + .anyMatch(apiName -> apiName.version().equals(BusinessMetricFeatureId.S3_EXPRESS_BUCKET.value())); + + assertThat(hasExistingApiName).isTrue(); + assertThat(hasS3ExpressApiName).isTrue(); + } + } +} diff --git a/services/ssooidc/src/test/java/software/amazon/awssdk/services/ssooidc/SsoLoginDeviceUserAgentTest.java b/services/ssooidc/src/test/java/software/amazon/awssdk/services/ssooidc/SsoLoginDeviceUserAgentTest.java new file mode 100644 index 000000000000..60da4d188e30 --- /dev/null +++ b/services/ssooidc/src/test/java/software/amazon/awssdk/services/ssooidc/SsoLoginDeviceUserAgentTest.java @@ -0,0 +1,96 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.ssooidc; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static software.amazon.awssdk.core.useragent.BusinessMetricCollection.METRIC_SEARCH_PATTERN; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import java.net.URI; +import java.time.Instant; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.ssooidc.internal.OnDiskTokenManager; +import software.amazon.awssdk.services.ssooidc.internal.SsoOidcToken; + +public class SsoLoginDeviceUserAgentTest { + + @Rule + public WireMockRule mockServer = new WireMockRule(0); + + private SsoOidcClient ssoOidcClient; + private static final String TEST_SESSION_NAME = "https://d-123.awsapps.com/start"; + + @Before + public void setup() { + ssoOidcClient = SsoOidcClient.builder() + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))) + .region(Region.US_WEST_2) + .endpointOverride(URI.create("http://localhost:" + mockServer.port())) + .build(); + } + + @Test + public void ssoOidcTokenProvider_shouldHaveSsoLoginDeviceUserAgent() { + // Setup mock response for SSO OIDC service + stubFor(any(urlEqualTo("/")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/x-amz-json-1.1") + .withBody("{\"access_token\":\"new-access-token\",\"expires_in\":3600}"))); + + // Create a test token that needs refreshing (expired) + OnDiskTokenManager tokenManager = OnDiskTokenManager.create(TEST_SESSION_NAME); + SsoOidcToken testToken = SsoOidcToken.builder() + .accessToken("test-access-token") + .clientId("test-client-id") + .clientSecret("test-client-secret") + .refreshToken("test-refresh-token") + .startUrl(TEST_SESSION_NAME) + .expiresAt(Instant.now().minusSeconds(10)) // Expired to force refresh + .build(); + tokenManager.storeToken(testToken); + + // Create SsoOidcTokenProvider and trigger token refresh + try (SsoOidcTokenProvider tokenProvider = SsoOidcTokenProvider.builder() + .sessionName(TEST_SESSION_NAME) + .ssoOidcClient(ssoOidcClient) + .build()) { + try { + // This will trigger a token refresh, which should include the SSO_LOGIN_DEVICE metric + tokenProvider.resolveToken(); + } catch (Exception e) { + // Expected - we're using a mock server, so the actual token refresh may fail + // But the important part is that the HTTP request was made with the correct User-Agent + } + } + + // Verify that the User-Agent header contains the SSO_LOGIN_DEVICE business metric + verify(postRequestedFor(urlEqualTo("/")).withHeader("User-Agent", + matching(METRIC_SEARCH_PATTERN.apply(BusinessMetricFeatureId.SSO_LOGIN_DEVICE.value())))); + } +}