Enhance AWS SDK Instrumentation with Detailed HTTP Error Information (#9448)

instrumentation/aws-sdk/README.md

@@ -5,7 +5,8 @@ For more information, see the respective public setters in the `AwsSdkTelemetryB
 - [SDK v1](./aws-sdk-1.11/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v1_11/AwsSdkTelemetryBuilder.java)
 - [SDK v2](./aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetryBuilder.java)
 
-| System property                                                          | Type    | Default | Description                                                                                                                           |
-| ------------------------------------------------------------------------ | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------- |
-| `otel.instrumentation.aws-sdk.experimental-span-attributes`              | Boolean | `false` | Enable the capture of experimental span attributes.                                                                                   |
+| System property                                                          | Type    | Default | Description                                                                                 |
+|--------------------------------------------------------------------------| ------- | ------- |---------------------------------------------------------------------------------------------|
+| `otel.instrumentation.aws-sdk.experimental-span-attributes`              | Boolean | `false` | Enable the capture of experimental span attributes.                                         |
 | `otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging` | Boolean | `false` | v2 only, inject into SNS/SQS attributes with configured propagator: See [v2 README](aws-sdk-2.2/library/README.md#trace-propagation). |
+| `otel.instrumentation.aws-sdk.experimental-record-individual-http-error` | Boolean | `false` | v2 only, record errors returned by each individual HTTP request as events for the SDK span. |

instrumentation/aws-sdk/aws-sdk-2.2/javaagent/build.gradle.kts

@@ -10,6 +10,7 @@ muzzle {
     // Used by all SDK services, the only case it isn't is an SDK extension such as a custom HTTP
     // client, which is not target of instrumentation anyways.
     extraDependency("software.amazon.awssdk:protocol-core")
+
     excludeInstrumentationName("aws-sdk-2.2-sqs")
     excludeInstrumentationName("aws-sdk-2.2-sns")
 
@@ -95,6 +96,7 @@ testing {
         } else {
           implementation("software.amazon.awssdk:s3:2.10.12")
         }
+        implementation(project(":instrumentation:aws-sdk:aws-sdk-2.2:library"))
       }
     }
   }
@@ -115,6 +117,7 @@ tasks {
   withType<Test>().configureEach {
     // TODO run tests both with and without experimental span attributes
     systemProperty("otel.instrumentation.aws-sdk.experimental-span-attributes", "true")
+    systemProperty("otel.instrumentation.aws-sdk.experimental-record-individual-http-error", "true")
   }
 
   withType<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>().configureEach {

instrumentation/aws-sdk/aws-sdk-2.2/javaagent/src/test/java/Aws2ClientRecordHttpErrorTest.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import io.opentelemetry.instrumentation.awssdk.v2_2.AbstractAws2ClientRecordHttpErrorTest;
+import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+
+public class Aws2ClientRecordHttpErrorTest extends AbstractAws2ClientRecordHttpErrorTest {
+  @RegisterExtension
+  private final AgentInstrumentationExtension testing = AgentInstrumentationExtension.create();
+
+  @Override
+  public ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder() {
+    return ClientOverrideConfiguration.builder();
+  }
+
+  @Override
+  protected InstrumentationExtension getTesting() {
+    return testing;
+  }
+}

instrumentation/aws-sdk/aws-sdk-2.2/library-autoconfigure/build.gradle.kts

@@ -25,5 +25,6 @@ tasks {
   test {
     systemProperty("otel.instrumentation.aws-sdk.experimental-span-attributes", true)
     systemProperty("otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging", true)
+    systemProperty("otel.instrumentation.aws-sdk.experimental-record-individual-http-error", true)
   }
 }

instrumentation/aws-sdk/aws-sdk-2.2/library-autoconfigure/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/autoconfigure/TracingExecutionInterceptor.java

@@ -36,10 +36,15 @@ public class TracingExecutionInterceptor implements ExecutionInterceptor {
       ConfigPropertiesUtil.getBoolean(
           "otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging", false);
 
+  private static final boolean RECORD_INDIVIDUAL_HTTP_ERROR =
+      ConfigPropertiesUtil.getBoolean(
+          "otel.instrumentation.aws-sdk.experimental-record-individual-http-error", false);
+
   private final ExecutionInterceptor delegate =
       AwsSdkTelemetry.builder(GlobalOpenTelemetry.get())
           .setCaptureExperimentalSpanAttributes(CAPTURE_EXPERIMENTAL_SPAN_ATTRIBUTES)
           .setUseConfiguredPropagatorForMessaging(USE_MESSAGING_PROPAGATOR)
+          .setRecordIndividualHttpError(RECORD_INDIVIDUAL_HTTP_ERROR)
           .build()
           .newExecutionInterceptor();
 

instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetry.java

@@ -46,12 +46,14 @@ public class AwsSdkTelemetry {
   private final boolean captureExperimentalSpanAttributes;
   @Nullable private final TextMapPropagator messagingPropagator;
   private final boolean useXrayPropagator;
+  private final boolean recordIndividualHttpError;
 
   AwsSdkTelemetry(
       OpenTelemetry openTelemetry,
       boolean captureExperimentalSpanAttributes,
       boolean useMessagingPropagator,
-      boolean useXrayPropagator) {
+      boolean useXrayPropagator,
+      boolean recordIndividualHttpError) {
     this.useXrayPropagator = useXrayPropagator;
     this.requestInstrumenter =
         AwsSdkInstrumenterFactory.requestInstrumenter(
@@ -62,6 +64,7 @@ public class AwsSdkTelemetry {
     this.captureExperimentalSpanAttributes = captureExperimentalSpanAttributes;
     this.messagingPropagator =
         useMessagingPropagator ? openTelemetry.getPropagators().getTextMapPropagator() : null;
+    this.recordIndividualHttpError = recordIndividualHttpError;
   }
 
   /**
@@ -74,6 +77,7 @@ public class AwsSdkTelemetry {
         consumerInstrumenter,
         captureExperimentalSpanAttributes,
         messagingPropagator,
-        useXrayPropagator);
+        useXrayPropagator,
+        recordIndividualHttpError);
   }
 }

instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetryBuilder.java

@@ -17,6 +17,8 @@ public final class AwsSdkTelemetryBuilder {
 
   private boolean useMessagingPropagator;
 
+  private boolean recordIndividualHttpError;
+
   private boolean useXrayPropagator = true;
 
   AwsSdkTelemetryBuilder(OpenTelemetry openTelemetry) {
@@ -57,6 +59,20 @@ public final class AwsSdkTelemetryBuilder {
     return this;
   }
 
+  /**
+   * Sets whether errors returned by each individual HTTP request should be recorded as events for
+   * the SDK span.
+   *
+   * <p>This option is off by default. If enabled, the HTTP error code and the error message will be
+   * captured and associated with the span. This provides detailed insights into errors on a
+   * per-request basis.
+   */
+  @CanIgnoreReturnValue
+  public AwsSdkTelemetryBuilder setRecordIndividualHttpError(boolean recordIndividualHttpError) {
+    this.recordIndividualHttpError = recordIndividualHttpError;
+    return this;
+  }
+
   /**
    * This setter implemented package-private for testing the messaging propagator, it does not seem
    * too useful in general. The option is on by default.
@@ -79,6 +95,7 @@ public final class AwsSdkTelemetryBuilder {
         openTelemetry,
         captureExperimentalSpanAttributes,
         useMessagingPropagator,
-        useXrayPropagator);
+        useXrayPropagator,
+        recordIndividualHttpError);
   }
 }

instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/TracingExecutionInterceptor.java

@@ -7,6 +7,7 @@ package io.opentelemetry.instrumentation.awssdk.v2_2;
 
 import static io.opentelemetry.instrumentation.awssdk.v2_2.AwsSdkRequestType.DYNAMODB;
 
+import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.api.trace.Span;
@@ -15,7 +16,14 @@ import io.opentelemetry.context.propagation.TextMapPropagator;
 import io.opentelemetry.contrib.awsxray.propagator.AwsXrayPropagator;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
 import io.opentelemetry.semconv.SemanticAttributes;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute;
 import software.amazon.awssdk.awscore.AwsResponse;
@@ -50,6 +58,10 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
   private final Instrumenter<ExecutionAttributes, SdkHttpResponse> consumerInstrumenter;
   private final boolean captureExperimentalSpanAttributes;
 
+  static final AttributeKey<String> HTTP_ERROR_MSG =
+      AttributeKey.stringKey("aws.http.error_message");
+  static final String HTTP_FAILURE_EVENT = "HTTP request failure";
+
   Instrumenter<ExecutionAttributes, SdkHttpResponse> getConsumerInstrumenter() {
     return consumerInstrumenter;
   }
@@ -65,6 +77,7 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
 
   @Nullable private final TextMapPropagator messagingPropagator;
   private final boolean useXrayPropagator;
+  private final boolean recordIndividualHttpError;
   private final FieldMapper fieldMapper;
 
   TracingExecutionInterceptor(
@@ -72,12 +85,14 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
       Instrumenter<ExecutionAttributes, SdkHttpResponse> consumerInstrumenter,
       boolean captureExperimentalSpanAttributes,
       TextMapPropagator messagingPropagator,
-      boolean useXrayPropagator) {
+      boolean useXrayPropagator,
+      boolean recordIndividualHttpError) {
     this.requestInstrumenter = requestInstrumenter;
     this.consumerInstrumenter = consumerInstrumenter;
     this.captureExperimentalSpanAttributes = captureExperimentalSpanAttributes;
     this.messagingPropagator = messagingPropagator;
     this.useXrayPropagator = useXrayPropagator;
+    this.recordIndividualHttpError = recordIndividualHttpError;
     this.fieldMapper = new FieldMapper();
   }
 
@@ -222,6 +237,19 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
     return builder.build();
   }
 
+  @Override
+  public Optional<InputStream> modifyHttpResponseContent(
+      Context.ModifyHttpResponse context, ExecutionAttributes executionAttributes) {
+    Optional<InputStream> responseBody = context.responseBody();
+    if (recordIndividualHttpError) {
+      String errorMsg = extractHttpErrorAsEvent(context, executionAttributes);
+      if (errorMsg != null) {
+        return Optional.of(new ByteArrayInputStream(errorMsg.getBytes(Charset.defaultCharset())));
+      }
+    }
+    return responseBody;
+  }
+
   private void populateRequestAttributes(
       Span span,
       AwsSdkRequest awsSdkRequest,
@@ -289,6 +317,37 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
     }
   }
 
+  private static String extractHttpErrorAsEvent(
+      Context.AfterTransmission context, ExecutionAttributes executionAttributes) {
+    io.opentelemetry.context.Context otelContext = getContext(executionAttributes);
+    if (otelContext != null) {
+      Span span = Span.fromContext(otelContext);
+      SdkHttpResponse response = context.httpResponse();
+
+      if (response != null && !response.isSuccessful()) {
+        int errorCode = response.statusCode();
+        // we want to record the error message from http response
+        Optional<InputStream> responseBody = context.responseBody();
+        if (responseBody.isPresent()) {
+          String errorMsg =
+              new BufferedReader(
+                      new InputStreamReader(responseBody.get(), Charset.defaultCharset()))
+                  .lines()
+                  .collect(Collectors.joining("\n"));
+          Attributes attributes =
+              Attributes.of(
+                  SemanticAttributes.HTTP_RESPONSE_STATUS_CODE,
+                  Long.valueOf(errorCode),
+                  HTTP_ERROR_MSG,
+                  errorMsg);
+          span.addEvent(HTTP_FAILURE_EVENT, attributes);
+          return errorMsg;
+        }
+      }
+    }
+    return null;
+  }
+
   @Override
   public void onExecutionFailure(
       Context.FailedExecution context, ExecutionAttributes executionAttributes) {

instrumentation/aws-sdk/aws-sdk-2.2/library/src/test/java/io/opentelemetry/instrumentation/awssdk/v2_2/Aws2ClientNotRecordHttpErrorTest.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.awssdk.v2_2;
+
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+
+public class Aws2ClientNotRecordHttpErrorTest extends AbstractAws2ClientRecordHttpErrorTest {
+  @RegisterExtension
+  public final LibraryInstrumentationExtension testing = LibraryInstrumentationExtension.create();
+
+  @Override
+  public ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder() {
+    return ClientOverrideConfiguration.builder()
+        .addExecutionInterceptor(
+            AwsSdkTelemetry.builder(testing.getOpenTelemetry())
+                .setCaptureExperimentalSpanAttributes(true)
+                .setRecordIndividualHttpError(isRecordIndividualHttpErrorEnabled())
+                .build()
+                .newExecutionInterceptor());
+  }
+
+  @Override
+  public boolean isRecordIndividualHttpErrorEnabled() {
+    return false;
+  }
+
+  @Override
+  protected InstrumentationExtension getTesting() {
+    return testing;
+  }
+}

instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2ClientRecordHttpErrorTest.java

@@ -0,0 +1,209 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.awssdk.v2_2;
+
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.instrumentation.api.internal.ConfigPropertiesUtil;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.semconv.SemanticAttributes;
+import io.opentelemetry.testing.internal.armeria.common.HttpResponse;
+import io.opentelemetry.testing.internal.armeria.common.HttpStatus;
+import io.opentelemetry.testing.internal.armeria.common.MediaType;
+import io.opentelemetry.testing.internal.armeria.testing.junit5.server.mock.MockWebServerExtension;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder;
+import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class AbstractAws2ClientRecordHttpErrorTest {
+  private static final StaticCredentialsProvider CREDENTIALS_PROVIDER =
+      StaticCredentialsProvider.create(
+          AwsBasicCredentials.create("my-access-key", "my-secret-key"));
+
+  private static final MockWebServerExtension server = new MockWebServerExtension();
+  protected static List<String> httpErrorMessages = new ArrayList<>();
+
+  @BeforeAll
+  public static void setupSpec() {
+    server.start();
+  }
+
+  public static void cleanupSpec() {
+    server.stop();
+  }
+
+  public abstract ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder();
+
+  protected abstract InstrumentationExtension getTesting();
+
+  // Introducing a new ExecutionInterceptor that's registered with the AWS SDK.
+  // It's positioned to execute after the TracingExecutionInterceptor used for SDK instrumentation.
+  // The purpose of this interceptor is to inspect the response body of failed HTTP requests.
+  // We aim to ensure that the HTTP error message remains accessible in the response body's
+  // InputStream
+  // even after the TracingExecutionInterceptor has processed it.
+  static class ResponseCheckInterceptor implements ExecutionInterceptor {
+    @Override
+    public Optional<InputStream> modifyHttpResponseContent(
+        Context.ModifyHttpResponse context, ExecutionAttributes executionAttributes) {
+      Optional<InputStream> responseBody = context.responseBody();
+      String errorMsg = extractHttpErrorMessage(context, executionAttributes);
+      if (errorMsg != null) {
+        return Optional.of(new ByteArrayInputStream(errorMsg.getBytes(Charset.defaultCharset())));
+      }
+      return responseBody;
+    }
+
+    private static String extractHttpErrorMessage(
+        Context.AfterTransmission context, ExecutionAttributes executionAttributes) {
+      SdkHttpResponse response = context.httpResponse();
+      if (executionAttributes == null) {
+        return "";
+      }
+
+      if (response != null && !response.isSuccessful()) {
+        Optional<InputStream> responseBody = context.responseBody();
+        if (responseBody.isPresent()) {
+          String errorMsg =
+              new BufferedReader(
+                      new InputStreamReader(responseBody.get(), Charset.defaultCharset()))
+                  .lines()
+                  .collect(Collectors.joining("\n"));
+          httpErrorMessages.add(errorMsg);
+          return errorMsg;
+        }
+      }
+      return null;
+    }
+  }
+
+  private static void cleanResponses() {
+    httpErrorMessages.clear();
+  }
+
+  public boolean isRecordIndividualHttpErrorEnabled() {
+    // See io.opentelemetry.instrumentation.awssdk.v2_2.autoconfigure.TracingExecutionInterceptor
+    return ConfigPropertiesUtil.getBoolean(
+        "otel.instrumentation.aws-sdk.experimental-record-individual-http-error", false);
+  }
+
+  @Test
+  // Suppressing deprecation because we use some deprecated attributes in the test
+  @SuppressWarnings("deprecation")
+  public void testSendDynamoDbRequestWithRetries() {
+    cleanResponses();
+    // Setup and configuration
+    String service = "DynamoDb";
+    String operation = "PutItem";
+    String method = "POST";
+    String requestId = "UNKNOWN";
+    DynamoDbClientBuilder builder = DynamoDbClient.builder();
+    ClientOverrideConfiguration.Builder overrideConfigBuilder =
+        createOverrideConfigurationBuilder()
+            .addExecutionInterceptor(new ResponseCheckInterceptor());
+    builder.overrideConfiguration(overrideConfigBuilder.build());
+
+    DynamoDbClient client =
+        builder
+            .endpointOverride(server.httpUri())
+            .region(Region.AP_NORTHEAST_1)
+            .credentialsProvider(CREDENTIALS_PROVIDER)
+            .build();
+
+    // Mocking server responses
+    server.enqueue(
+        HttpResponse.of(
+            HttpStatus.INTERNAL_SERVER_ERROR,
+            MediaType.PLAIN_TEXT_UTF_8,
+            "DynamoDB could not process your request"));
+    server.enqueue(
+        HttpResponse.of(
+            HttpStatus.SERVICE_UNAVAILABLE,
+            MediaType.PLAIN_TEXT_UTF_8,
+            "DynamoDB is currently unavailable"));
+    server.enqueue(HttpResponse.of(HttpStatus.OK, MediaType.PLAIN_TEXT_UTF_8, ""));
+
+    // Making the call
+    client.putItem(PutItemRequest.builder().tableName("sometable").build());
+
+    getTesting()
+        .waitAndAssertTraces(
+            trace -> {
+              trace.hasSpansSatisfyingExactly(
+                  span -> {
+                    span.hasKind(SpanKind.CLIENT);
+                    span.hasNoParent();
+                    span.hasAttributesSatisfying(
+                        attributes -> {
+                          assertThat(attributes)
+                              .containsEntry(SemanticAttributes.NET_PEER_NAME, "127.0.0.1")
+                              .containsEntry(SemanticAttributes.NET_PEER_PORT, server.httpPort())
+                              .containsEntry(SemanticAttributes.HTTP_METHOD, method)
+                              .containsEntry(SemanticAttributes.HTTP_STATUS_CODE, 200)
+                              .containsEntry(SemanticAttributes.RPC_SYSTEM, "aws-api")
+                              .containsEntry(SemanticAttributes.RPC_SERVICE, service)
+                              .containsEntry(SemanticAttributes.RPC_METHOD, operation)
+                              .containsEntry("aws.agent", "java-aws-sdk")
+                              .containsEntry("aws.requestId", requestId)
+                              .containsEntry("aws.table.name", "sometable")
+                              .containsEntry(SemanticAttributes.DB_SYSTEM, "dynamodb")
+                              .containsEntry(SemanticAttributes.DB_OPERATION, operation);
+                        });
+                    if (isRecordIndividualHttpErrorEnabled()) {
+                      span.hasEventsSatisfyingExactly(
+                          event ->
+                              event
+                                  .hasName("HTTP request failure")
+                                  .hasAttributesSatisfyingExactly(
+                                      equalTo(SemanticAttributes.HTTP_RESPONSE_STATUS_CODE, 500),
+                                      equalTo(
+                                          AttributeKey.stringKey("aws.http.error_message"),
+                                          "DynamoDB could not process your request")),
+                          event ->
+                              event
+                                  .hasName("HTTP request failure")
+                                  .hasAttributesSatisfyingExactly(
+                                      equalTo(SemanticAttributes.HTTP_RESPONSE_STATUS_CODE, 503),
+                                      equalTo(
+                                          AttributeKey.stringKey("aws.http.error_message"),
+                                          "DynamoDB is currently unavailable")));
+                    } else {
+                      span.hasEventsSatisfying(events -> assertThat(events.size()).isEqualTo(0));
+                    }
+                  });
+            });
+
+    // make sure the response body input stream is still available and check its content to be
+    // expected
+    assertThat(httpErrorMessages.size()).isEqualTo(2);
+    assertThat(httpErrorMessages.get(0)).isEqualTo("DynamoDB could not process your request");
+    assertThat(httpErrorMessages.get(1)).isEqualTo("DynamoDB is currently unavailable");
+  }
+}