Implement @WithSpan support for kotlin coroutines (#8870)

Co-authored-by: Trask Stalnaker <trask.stalnaker@gmail.com>

dependencyManagement/build.gradle.kts

@@ -65,6 +65,7 @@ val CORE_DEPENDENCIES = listOf(
   "net.bytebuddy:byte-buddy-gradle-plugin:${byteBuddyVersion}",
   "org.ow2.asm:asm:${asmVersion}",
   "org.ow2.asm:asm-tree:${asmVersion}",
+  "org.ow2.asm:asm-util:${asmVersion}",
   "org.openjdk.jmh:jmh-core:${jmhVersion}",
   "org.openjdk.jmh:jmh-generator-bytecode:${jmhVersion}",
   "org.mockito:mockito-core:${mockitoVersion}",

instrumentation/kotlinx-coroutines/javaagent/build.gradle.kts

@@ -10,18 +10,27 @@ muzzle {
     group.set("org.jetbrains.kotlinx")
     module.set("kotlinx-coroutines-core")
     versions.set("[1.0.0,1.3.8)")
+    extraDependency(project(":instrumentation-annotations"))
+    extraDependency("io.opentelemetry:opentelemetry-api:1.27.0")
   }
   // 1.3.9 (and beyond?) have changed how artifact names are resolved due to multiplatform variants
   pass {
     group.set("org.jetbrains.kotlinx")
     module.set("kotlinx-coroutines-core-jvm")
     versions.set("[1.3.9,)")
+    extraDependency(project(":instrumentation-annotations"))
+    extraDependency("io.opentelemetry:opentelemetry-api:1.27.0")
   }
 }
 
 dependencies {
   compileOnly("io.opentelemetry:opentelemetry-extension-kotlin")
   compileOnly("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
+  compileOnly(project(":opentelemetry-instrumentation-annotations-shaded-for-instrumenting", configuration = "shadow"))
+
+  implementation("org.ow2.asm:asm-tree")
+  implementation("org.ow2.asm:asm-util")
+  implementation(project(":instrumentation:opentelemetry-instrumentation-annotations-1.16:javaagent"))
 
   testInstrumentation(project(":instrumentation:opentelemetry-extension-kotlin-1.0:javaagent"))
   testInstrumentation(project(":instrumentation:reactor:reactor-3.1:javaagent"))
@@ -29,6 +38,7 @@ dependencies {
   testImplementation("io.opentelemetry:opentelemetry-extension-kotlin")
   testImplementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
   testImplementation(project(":instrumentation:reactor:reactor-3.1:library"))
+  testImplementation(project(":instrumentation-annotations"))
 
   // Use first version with flow support since we have tests for it.
   testLibrary("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0")
@@ -39,6 +49,8 @@ tasks {
   withType(KotlinCompile::class).configureEach {
     kotlinOptions {
       jvmTarget = "1.8"
+      // generate metadata for Java 1.8 reflection on method parameters, used in @WithSpan tests
+      javaParameters = true
     }
   }
 }

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/AnnotationInstrumentationHelper.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import static io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations.AnnotationSingletons.instrumenter;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
+import kotlin.coroutines.Continuation;
+import kotlin.coroutines.intrinsics.IntrinsicsKt;
+
+public final class AnnotationInstrumentationHelper {
+
+  private static final VirtualField<Continuation<?>, Context> contextField =
+      VirtualField.find(Continuation.class, Context.class);
+
+  public static MethodRequest createMethodRequest(
+      Class<?> declaringClass, String methodName, String withSpanValue, String spanKindString) {
+    SpanKind spanKind = SpanKind.INTERNAL;
+    if (spanKindString != null) {
+      try {
+        spanKind = SpanKind.valueOf(spanKindString);
+      } catch (IllegalArgumentException exception) {
+        // ignore
+      }
+    }
+
+    return MethodRequest.create(declaringClass, methodName, withSpanValue, spanKind);
+  }
+
+  public static Context enterCoroutine(
+      int label, Continuation<?> continuation, MethodRequest request) {
+    // label 0 means that coroutine is started, any other label means that coroutine is resumed
+    if (label == 0) {
+      Context context = instrumenter().start(Context.current(), request);
+      // null continuation means that this method is not going to be resumed, and we don't need to
+      // store the context
+      if (continuation != null) {
+        contextField.set(continuation, context);
+      }
+      return context;
+    } else {
+      return continuation != null ? contextField.get(continuation) : null;
+    }
+  }
+
+  public static Scope openScope(Context context) {
+    return context != null ? context.makeCurrent() : null;
+  }
+
+  public static void exitCoroutine(
+      Object result,
+      MethodRequest request,
+      Continuation<?> continuation,
+      Context context,
+      Scope scope) {
+    exitCoroutine(null, result, request, continuation, context, scope);
+  }
+
+  public static void exitCoroutine(
+      Throwable error,
+      Object result,
+      MethodRequest request,
+      Continuation<?> continuation,
+      Context context,
+      Scope scope) {
+    if (scope == null) {
+      return;
+    }
+    scope.close();
+
+    // end the span when this method can not be resumed (coroutine is null) or if it has reached
+    // final state (returns anything else besides COROUTINE_SUSPENDED)
+    if (continuation == null || result != IntrinsicsKt.getCOROUTINE_SUSPENDED()) {
+      instrumenter().end(context, request, null, error);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, boolean value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, byte value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, char value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, String.valueOf(value));
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, double value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, float value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, int value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, long value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, short value) {
+    // only add the attribute when coroutine is started
+    if (label == 0) {
+      Span.current().setAttribute(name, value);
+    }
+  }
+
+  public static void setSpanAttribute(int label, String name, Object value) {
+    // only add the attribute when coroutine is started
+    if (label != 0) {
+      return;
+    }
+    if (value instanceof String) {
+      Span.current().setAttribute(name, (String) value);
+    } else if (value instanceof Boolean) {
+      Span.current().setAttribute(name, (Boolean) value);
+    } else if (value instanceof Byte) {
+      Span.current().setAttribute(name, (Byte) value);
+    } else if (value instanceof Character) {
+      Span.current().setAttribute(name, (Character) value);
+    } else if (value instanceof Double) {
+      Span.current().setAttribute(name, (Double) value);
+    } else if (value instanceof Float) {
+      Span.current().setAttribute(name, (Float) value);
+    } else if (value instanceof Integer) {
+      Span.current().setAttribute(name, (Integer) value);
+    } else if (value instanceof Long) {
+      Span.current().setAttribute(name, (Long) value);
+    }
+    // TODO: arrays and List not supported see AttributeBindingFactoryTest
+  }
+
+  public static void init() {}
+
+  private AnnotationInstrumentationHelper() {}
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/AnnotationInstrumentationModule.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.hasClassesNamed;
+import static java.util.Collections.singletonList;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import java.util.List;
+import net.bytebuddy.matcher.ElementMatcher;
+
+/** Instrumentation for methods annotated with {@code WithSpan} annotation. */
+@AutoService(InstrumentationModule.class)
+public class AnnotationInstrumentationModule extends InstrumentationModule {
+
+  public AnnotationInstrumentationModule() {
+    super(
+        "kotlinx-coroutines-opentelemetry-instrumentation-annotations",
+        "kotlinx-coroutines",
+        "opentelemetry-instrumentation-annotations");
+  }
+
+  @Override
+  public int order() {
+    // Run first to ensure other automatic instrumentation is added after and therefore is executed
+    // earlier in the instrumented method and create the span to attach attributes to.
+    return -1000;
+  }
+
+  @Override
+  public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
+    return hasClassesNamed(
+        "application.io.opentelemetry.instrumentation.annotations.WithSpan",
+        "kotlinx.coroutines.CoroutineContextKt");
+  }
+
+  @Override
+  public List<TypeInstrumentation> typeInstrumentations() {
+    return singletonList(new WithSpanInstrumentation());
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/AnnotationSingletons.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.code.CodeAttributesExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.util.SpanNames;
+
+public final class AnnotationSingletons {
+
+  private static final String INSTRUMENTATION_NAME = "io.opentelemetry.kotlinx-coroutines";
+
+  private static final Instrumenter<MethodRequest, Object> INSTRUMENTER = createInstrumenter();
+
+  public static Instrumenter<MethodRequest, Object> instrumenter() {
+    return INSTRUMENTER;
+  }
+
+  private static Instrumenter<MethodRequest, Object> createInstrumenter() {
+    return Instrumenter.builder(
+            GlobalOpenTelemetry.get(),
+            INSTRUMENTATION_NAME,
+            AnnotationSingletons::spanNameFromMethodRequest)
+        .addAttributesExtractor(
+            CodeAttributesExtractor.create(MethodRequestCodeAttributesGetter.INSTANCE))
+        .buildInstrumenter(MethodRequest::getSpanKind);
+  }
+
+  private static String spanNameFromMethodRequest(MethodRequest request) {
+    String spanName = request.getWithSpanValue();
+    if (spanName == null || spanName.isEmpty()) {
+      spanName = SpanNames.fromMethod(request.getDeclaringClass(), request.getMethodName());
+    }
+    return spanName;
+  }
+
+  private AnnotationSingletons() {}
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/ExpandFramesClassVisitor.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+/**
+ * Converts compressed frames (F_FULL, F_SAME etc.) into expanded frames (F_NEW). Using this visitor
+ * should give the same result as using ClassReader.EXPAND_FRAMES.
+ */
+class ExpandFramesClassVisitor extends ClassVisitor {
+  private String className;
+
+  ExpandFramesClassVisitor(ClassVisitor classVisitor) {
+    super(Opcodes.ASM9, classVisitor);
+  }
+
+  @Override
+  public void visit(
+      int version,
+      int access,
+      String name,
+      String signature,
+      String superName,
+      String[] interfaces) {
+    super.visit(version, access, name, signature, superName, interfaces);
+    className = name;
+  }
+
+  @Override
+  public MethodVisitor visitMethod(
+      int access, String name, String descriptor, String signature, String[] exceptions) {
+    MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+    return new ExpandFramesMethodVisitor(mv, className, access, descriptor);
+  }
+
+  private static class ExpandFramesMethodVisitor extends MethodVisitor {
+    final List<Object> currentLocals = new ArrayList<>();
+    final List<Object> currentStack = new ArrayList<>();
+
+    ExpandFramesMethodVisitor(MethodVisitor mv, String className, int access, String descriptor) {
+      super(Opcodes.ASM9, mv);
+      if (!Modifier.isStatic(access)) {
+        currentLocals.add(className);
+      }
+      for (Type type : Type.getArgumentTypes(descriptor)) {
+        switch (type.getSort()) {
+          case Type.BOOLEAN:
+          case Type.BYTE:
+          case Type.CHAR:
+          case Type.INT:
+          case Type.SHORT:
+            currentLocals.add(Opcodes.INTEGER);
+            break;
+          case Type.DOUBLE:
+            currentLocals.add(Opcodes.DOUBLE);
+            break;
+          case Type.FLOAT:
+            currentLocals.add(Opcodes.FLOAT);
+            break;
+          case Type.LONG:
+            currentLocals.add(Opcodes.LONG);
+            break;
+          case Type.ARRAY:
+          case Type.OBJECT:
+            currentLocals.add(type.getInternalName());
+            break;
+          default:
+            throw new IllegalStateException("Unexpected type " + type.getSort() + " " + type);
+        }
+      }
+    }
+
+    private static void copy(Object[] array, int count, List<Object> list) {
+      list.clear();
+      for (int i = 0; i < count; i++) {
+        list.add(array[i]);
+      }
+    }
+
+    @Override
+    public void visitFrame(int type, int numLocal, Object[] local, int numStack, Object[] stack) {
+      switch (type) {
+          // An expanded frame.
+        case Opcodes.F_NEW:
+          // A compressed frame with complete frame data.
+        case Opcodes.F_FULL:
+          copy(local, numLocal, currentLocals);
+          copy(stack, numStack, currentStack);
+          break;
+          // A compressed frame with exactly the same locals as the previous frame and with an empty
+          // stack.
+        case Opcodes.F_SAME:
+          currentStack.clear();
+          break;
+          // A compressed frame with exactly the same locals as the previous frame and with a single
+          // value on the stack.
+        case Opcodes.F_SAME1:
+          currentStack.clear();
+          currentStack.add(stack[0]);
+          break;
+          // A compressed frame where locals are the same as the locals in the previous frame,
+          // except that additional 1-3 locals are defined, and with an empty stack.
+        case Opcodes.F_APPEND:
+          currentStack.clear();
+          for (int i = 0; i < numLocal; i++) {
+            currentLocals.add(local[i]);
+          }
+          break;
+          // A compressed frame where locals are the same as the locals in the previous frame,
+          // except that the last 1-3 locals are absent and with an empty stack.
+        case Opcodes.F_CHOP:
+          currentStack.clear();
+          for (Iterator<Object> iterator =
+                  currentLocals.listIterator(currentLocals.size() - numLocal);
+              iterator.hasNext(); ) {
+            iterator.next();
+            iterator.remove();
+          }
+          break;
+        default:
+          throw new IllegalStateException("Unexpected frame type " + type);
+      }
+
+      // visit expanded frame
+      super.visitFrame(
+          Opcodes.F_NEW,
+          currentLocals.size(),
+          currentLocals.toArray(),
+          currentStack.size(),
+          currentStack.toArray());
+    }
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/KotlinCoroutinesIgnoredTypesConfigurer.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.extension.ignore.IgnoredTypesBuilder;
+import io.opentelemetry.javaagent.extension.ignore.IgnoredTypesConfigurer;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+
+@AutoService(IgnoredTypesConfigurer.class)
+public class KotlinCoroutinesIgnoredTypesConfigurer implements IgnoredTypesConfigurer {
+
+  @Override
+  public void configure(IgnoredTypesBuilder builder, ConfigProperties config) {
+    builder.allowClass("kotlin.coroutines.jvm.internal.CompletedContinuation");
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/MethodRequest.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import io.opentelemetry.api.trace.SpanKind;
+
+public final class MethodRequest {
+  private final Class<?> declaringClass;
+  private final String methodName;
+  private final String withSpanValue;
+  private final SpanKind spanKind;
+
+  private MethodRequest(
+      Class<?> declaringClass, String methodName, String withSpanValue, SpanKind spanKind) {
+    this.declaringClass = declaringClass;
+    this.methodName = methodName;
+    this.withSpanValue = withSpanValue;
+    this.spanKind = spanKind;
+  }
+
+  public static MethodRequest create(
+      Class<?> declaringClass, String methodName, String withSpanValue, SpanKind spanKind) {
+    return new MethodRequest(declaringClass, methodName, withSpanValue, spanKind);
+  }
+
+  public Class<?> getDeclaringClass() {
+    return declaringClass;
+  }
+
+  public String getMethodName() {
+    return methodName;
+  }
+
+  public String getWithSpanValue() {
+    return withSpanValue;
+  }
+
+  public SpanKind getSpanKind() {
+    return spanKind;
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/MethodRequestCodeAttributesGetter.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import io.opentelemetry.instrumentation.api.instrumenter.code.CodeAttributesGetter;
+
+enum MethodRequestCodeAttributesGetter implements CodeAttributesGetter<MethodRequest> {
+  INSTANCE;
+
+  @Override
+  public Class<?> getCodeClass(MethodRequest methodRequest) {
+    return methodRequest.getDeclaringClass();
+  }
+
+  @Override
+  public String getMethodName(MethodRequest methodRequest) {
+    return methodRequest.getMethodName();
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/SpanAttributeUtil.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.tree.AnnotationNode;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.ParameterNode;
+
+class SpanAttributeUtil {
+
+  static class Parameter {
+    final int var;
+    final String name;
+    final Type type;
+
+    Parameter(int var, String name, Type type) {
+      this.var = var;
+      this.name = name;
+      this.type = type;
+    }
+  }
+
+  /**
+   * Collect method parameters with @SpanAttribute annotation. Span attribute is named based on the
+   * value of the annotation or using the parameter name in the source code, if neither is set then
+   * the parameter is ignored.
+   */
+  static List<Parameter> collectAnnotatedParameters(MethodNode source) {
+    List<Parameter> annotatedParameters = new ArrayList<>();
+    if (source.visibleParameterAnnotations != null) {
+      int slot = 1; // this is in slot 0
+      Type[] parameterTypes = Type.getArgumentTypes(source.desc);
+      for (int i = 0; i < parameterTypes.length; i++) {
+        Type type = parameterTypes[i];
+        // if current parameter index is equal or larger than the count of annotated parameters
+        // we have already checked all the parameters with annotations
+        if (i >= source.visibleParameterAnnotations.length) {
+          break;
+        }
+        boolean hasSpanAttributeAnnotation = false;
+        String name = getParameterName(source, i);
+        List<AnnotationNode> parameterAnnotations = source.visibleParameterAnnotations[i];
+        if (parameterAnnotations != null) {
+          for (AnnotationNode annotationNode : parameterAnnotations) {
+            if ("Lapplication/io/opentelemetry/instrumentation/annotations/SpanAttribute;"
+                .equals(annotationNode.desc)) {
+              // check whether SpanAttribute annotation has a value, if it has use that as
+              // parameter name
+              Object attributeValue = getAnnotationValue(annotationNode);
+              if (attributeValue instanceof String) {
+                name = (String) attributeValue;
+              }
+
+              hasSpanAttributeAnnotation = true;
+              break;
+            }
+          }
+        }
+        if (hasSpanAttributeAnnotation && name != null) {
+          annotatedParameters.add(new Parameter(slot, name, type));
+        }
+        slot += type.getSize();
+      }
+    }
+
+    return annotatedParameters;
+  }
+
+  private static String getParameterName(MethodNode methodNode, int parameter) {
+    ParameterNode parameterNode =
+        methodNode.parameters != null && methodNode.parameters.size() > parameter
+            ? methodNode.parameters.get(parameter)
+            : null;
+    return parameterNode != null ? parameterNode.name : null;
+  }
+
+  private static Object getAnnotationValue(AnnotationNode annotationNode) {
+    if (annotationNode.values != null && !annotationNode.values.isEmpty()) {
+      List<Object> values = annotationNode.values;
+      for (int j = 0; j < values.size(); j += 2) {
+        String attributeName = (String) values.get(j);
+        Object attributeValue = values.get(j + 1);
+        if ("value".equals(attributeName)) {
+          return attributeValue;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private SpanAttributeUtil() {}
+}

instrumentation/kotlinx-coroutines/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/instrumentationannotations/WithSpanInstrumentation.java

@@ -0,0 +1,521 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations;
+
+import static io.opentelemetry.javaagent.instrumentation.instrumentationannotations.KotlinCoroutineUtil.isKotlinSuspendMethod;
+import static net.bytebuddy.matcher.ElementMatchers.declaresMethod;
+import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
+import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.none;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.javaagent.bootstrap.internal.InstrumentationConfig;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
+import io.opentelemetry.javaagent.instrumentation.instrumentationannotations.AnnotationExcludedMethods;
+import io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines.instrumentationannotations.SpanAttributeUtil.Parameter;
+import java.util.Arrays;
+import java.util.List;
+import kotlin.coroutines.Continuation;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.asm.AsmVisitorWrapper;
+import net.bytebuddy.description.annotation.AnnotationSource;
+import net.bytebuddy.description.field.FieldDescription;
+import net.bytebuddy.description.field.FieldList;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.method.MethodList;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.implementation.Implementation;
+import net.bytebuddy.matcher.ElementMatcher;
+import net.bytebuddy.pool.TypePool;
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.GeneratorAdapter;
+import org.objectweb.asm.tree.AbstractInsnNode;
+import org.objectweb.asm.tree.AnnotationNode;
+import org.objectweb.asm.tree.FieldInsnNode;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.VarInsnNode;
+import org.objectweb.asm.util.CheckClassAdapter;
+
+class WithSpanInstrumentation implements TypeInstrumentation {
+  // whether to check the transformed bytecode with asm CheckClassAdapter
+  private static final boolean CHECK_CLASS =
+      InstrumentationConfig.get()
+          .getBoolean(
+              "otel.instrumentation.kotlinx-coroutines.check-class",
+              InstrumentationConfig.get().getBoolean("otel.javaagent.debug", false));
+
+  private final ElementMatcher.Junction<AnnotationSource> annotatedMethodMatcher;
+  // this matcher matches all methods that should be excluded from transformation
+  private final ElementMatcher.Junction<MethodDescription> excludedMethodsMatcher;
+
+  WithSpanInstrumentation() {
+    annotatedMethodMatcher =
+        isAnnotatedWith(named("application.io.opentelemetry.instrumentation.annotations.WithSpan"));
+    excludedMethodsMatcher = AnnotationExcludedMethods.configureExcludedMethods();
+  }
+
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return not(nameStartsWith("kotlin.coroutines."))
+        .and(
+            declaresMethod(
+                annotatedMethodMatcher
+                    .and(isKotlinSuspendMethod())
+                    .and(not(excludedMethodsMatcher))));
+  }
+
+  @Override
+  public void transform(TypeTransformer transformer) {
+    transformer.applyAdviceToMethod(
+        none(), WithSpanInstrumentation.class.getName() + "$InitAdvice");
+
+    transformer.applyTransformer(
+        (builder, typeDescription, classLoader, javaModule, protectionDomain) ->
+            builder.visit(
+                new AsmVisitorWrapper() {
+                  @Override
+                  public int mergeWriter(int flags) {
+                    return flags | ClassWriter.COMPUTE_MAXS;
+                  }
+
+                  @Override
+                  @CanIgnoreReturnValue
+                  public int mergeReader(int flags) {
+                    return flags;
+                  }
+
+                  @Override
+                  public ClassVisitor wrap(
+                      TypeDescription instrumentedType,
+                      ClassVisitor classVisitor,
+                      Implementation.Context implementationContext,
+                      TypePool typePool,
+                      FieldList<FieldDescription.InDefinedShape> fields,
+                      MethodList<?> methods,
+                      int writerFlags,
+                      int readerFlags) {
+                    if (CHECK_CLASS) {
+                      classVisitor = new CheckClassAdapter(classVisitor);
+                    }
+                    // we are using a visitor that converts compressed frames into expanded frames
+                    // because WithSpanClassVisitor uses GeneratorAdapter for adding new local
+                    // variables that requires expanded frames. We are not using
+                    // ClassReader.EXPAND_FRAMES because ExceptionHandlers class generates
+                    // compressed F_SAME frame that we can't easily replace with an expanded frame
+                    // because we don't know what locals are available at that point.
+                    return new ExpandFramesClassVisitor(new WithSpanClassVisitor(classVisitor));
+                  }
+                }));
+  }
+
+  @SuppressWarnings("unused")
+  public static class InitAdvice {
+
+    @Advice.OnMethodEnter(suppress = Throwable.class)
+    public static void onEnter() {
+      // this advice is here only to get AnnotationInstrumentationHelper injected
+      AnnotationInstrumentationHelper.init();
+    }
+  }
+
+  private static class WithSpanClassVisitor extends ClassVisitor {
+    String className;
+
+    WithSpanClassVisitor(ClassVisitor cv) {
+      super(Opcodes.ASM9, cv);
+    }
+
+    @Override
+    public void visit(
+        int version,
+        int access,
+        String name,
+        String signature,
+        String superName,
+        String[] interfaces) {
+      super.visit(version, access, name, signature, superName, interfaces);
+      className = name;
+    }
+
+    @Override
+    public MethodVisitor visitMethod(
+        int access, String name, String descriptor, String signature, String[] exceptions) {
+      MethodVisitor target = super.visitMethod(access, name, descriptor, signature, exceptions);
+      // firstly check whether this method could be a suspend method
+      // kotlin suspend methods take kotlin.coroutines.Continuation as last argument and return
+      // java.lang.Object
+      Type[] argumentTypes = Type.getArgumentTypes(descriptor);
+      if (argumentTypes.length > 0
+          && "kotlin/coroutines/Continuation"
+              .equals(argumentTypes[argumentTypes.length - 1].getInternalName())
+          && "java/lang/Object".equals(Type.getReturnType(descriptor).getInternalName())) {
+        // store method in MethodNode, so we could test whether it has the WithSpan annotation and
+        // depending on that either instrument it or leave it as it is
+        return new MethodNode(api, access, name, descriptor, signature, exceptions) {
+          @Override
+          public void visitEnd() {
+            super.visitEnd();
+
+            MethodVisitor mv = target;
+            if (hasWithSpanAnnotation(this)) {
+              mv = instrument(mv, this, className);
+            }
+            this.accept(mv);
+          }
+        };
+      }
+
+      return target;
+    }
+
+    private static boolean hasAnnotation(List<AnnotationNode> annotations, String annotationDesc) {
+      if (annotations != null) {
+        for (AnnotationNode annotationNode : annotations) {
+          if (annotationDesc.equals(annotationNode.desc)) {
+            return true;
+          }
+        }
+      }
+      return false;
+    }
+
+    private static boolean hasWithSpanAnnotation(MethodNode methodNode) {
+      return hasAnnotation(
+          methodNode.visibleAnnotations,
+          "Lapplication/io/opentelemetry/instrumentation/annotations/WithSpan;");
+    }
+
+    private static MethodVisitor instrument(
+        MethodVisitor target, MethodNode source, String className) {
+      // collect method arguments with @SpanAttribute annotation
+      List<Parameter> annotatedParameters = SpanAttributeUtil.collectAnnotatedParameters(source);
+
+      String methodName = source.name;
+      MethodNode methodNode =
+          new MethodNode(
+              source.access,
+              source.name,
+              source.desc,
+              source.signature,
+              source.exceptions.toArray(new String[0]));
+      GeneratorAdapter generatorAdapter =
+          new GeneratorAdapter(Opcodes.ASM9, methodNode, source.access, source.name, source.desc) {
+            int requestLocal;
+            int ourContinuationLocal;
+            int contextLocal;
+            int scopeLocal;
+            int lastLocal;
+
+            final Label start = new Label();
+            final Label handler = new Label();
+
+            String withSpanValue = null;
+            String spanKind = null;
+
+            @Override
+            public void visitCode() {
+              super.visitCode();
+              // add our local variables after method arguments, this will shift rest of the locals
+              requestLocal = newLocal(Type.getType(MethodRequest.class));
+              ourContinuationLocal = newLocal(Type.getType(Continuation.class));
+              contextLocal = newLocal(Type.getType(Context.class));
+              scopeLocal = newLocal(Type.getType(Scope.class));
+              // set lastLocal to the last local we added
+              lastLocal = scopeLocal;
+
+              visitLabel(start);
+            }
+
+            @Override
+            public void visitMaxs(int maxStack, int maxLocals) {
+              visitLabel(handler);
+              visitTryCatchBlock(start, handler, handler, null);
+              super.visitMaxs(maxStack, maxLocals);
+            }
+
+            @Override
+            public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
+              AnnotationVisitor annotationVisitor = super.visitAnnotation(descriptor, visible);
+              // remember value and kind from the @WithSpan annotation
+              if ("Lapplication/io/opentelemetry/instrumentation/annotations/WithSpan;"
+                  .equals(descriptor)) {
+                return new AnnotationVisitor(api, annotationVisitor) {
+                  @Override
+                  public void visit(String name, Object value) {
+                    if ("value".equals(name) && value instanceof String) {
+                      withSpanValue = (String) value;
+                    }
+                    super.visit(name, value);
+                  }
+
+                  @Override
+                  public void visitEnum(String name, String descriptor, String value) {
+                    if ("kind".equals(name)
+                        && "Lapplication/io/opentelemetry/api/trace/SpanKind;".equals(descriptor)) {
+                      spanKind = value;
+                    }
+                    super.visitEnum(name, descriptor, value);
+                  }
+                };
+              }
+              return annotationVisitor;
+            }
+
+            @Override
+            public void visitEnd() {
+              super.visitEnd();
+
+              // If a suspend method does not contain any blocking operations or has no code after
+              // the blocking operation it gets compiled to a regular method that we instrument the
+              // same way as the regular @WithSpan handling does. We create the span at the start of
+              // the method and end it in before every return instruction and in exception handler.
+              // If a suspend method has a blocking operation and code that needs to be executed
+              // after it, we start the span only when the coroutine was started, on resume we just
+              // activate the scope. We end the span when coroutine completes, otherwise we only
+              // close the scope.
+              // First we'll search for a bytecode sequence that looks like
+              // 64: aload         6
+              // 66: getfield      #444                // Field
+              // io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/KotlinCoroutinesInstrumentationTest$b2$1.label:I
+              // 69: tableswitch   { // 0 to 1
+              //                0: 92
+              //                1: 181
+              //          default: 210
+              // We are interested in the continuation local (here slot 6) and the value of the
+              // label field. To get the value of the label we'll insert our code between the
+              // getfield and tableswitch instructions.
+              int continuationLocal = -1;
+              AbstractInsnNode insertAfterInsn = null;
+              for (int i = 1; i < methodNode.instructions.size() - 1; i++) {
+                AbstractInsnNode instruction = methodNode.instructions.get(i);
+                if (instruction.getOpcode() == Opcodes.GETFIELD
+                    && "label".equals(((FieldInsnNode) instruction).name)
+                    && "I".equals(((FieldInsnNode) instruction).desc)) {
+                  if (methodNode.instructions.get(i + 1).getOpcode() != Opcodes.TABLESWITCH) {
+                    continue;
+                  }
+                  if (methodNode.instructions.get(i - 1).getOpcode() != Opcodes.ALOAD) {
+                    continue;
+                  }
+                  insertAfterInsn = instruction;
+                  continuationLocal = ((VarInsnNode) methodNode.instructions.get(i - 1)).var;
+                  break;
+                }
+              }
+
+              boolean hasBlockingOperation = insertAfterInsn != null && continuationLocal != -1;
+
+              // initialize our local variables, start span and open scope
+              {
+                MethodNode temp = new MethodNode();
+                // insert
+                // request =
+                // AnnotationInstrumentationHelper.createMethodRequest(InstrumentedClass.class,
+                //   instrumentedMethodName, withSpanValue, withSpanKind)
+                // context = AnnotationInstrumentationHelper.enterCoroutine(label, continuation,
+                // request)
+                // scope = AnnotationInstrumentationHelper.openScope(context)
+                if (hasBlockingOperation) {
+                  // value of label is on stack
+                  // label is used in call to enterCoroutine and later in @SpanAttribute handling
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitVarInsn(Opcodes.ALOAD, continuationLocal);
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitVarInsn(Opcodes.ASTORE, ourContinuationLocal);
+                } else {
+                  // nothing on stack, we are inserting code at the start of the method
+                  // we'll use 0 for label and null for continuation object
+                  temp.visitInsn(Opcodes.ICONST_0);
+                  temp.visitInsn(Opcodes.ICONST_0);
+                  temp.visitInsn(Opcodes.ACONST_NULL);
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitVarInsn(Opcodes.ASTORE, ourContinuationLocal);
+                }
+                temp.visitLdcInsn(Type.getObjectType(className));
+                temp.visitLdcInsn(methodName);
+                if (withSpanValue != null) {
+                  temp.visitLdcInsn(withSpanValue);
+                } else {
+                  temp.visitInsn(Opcodes.ACONST_NULL);
+                }
+                if (spanKind != null) {
+                  temp.visitLdcInsn(spanKind);
+                } else {
+                  temp.visitInsn(Opcodes.ACONST_NULL);
+                }
+                temp.visitMethodInsn(
+                    Opcodes.INVOKESTATIC,
+                    Type.getInternalName(AnnotationInstrumentationHelper.class),
+                    "createMethodRequest",
+                    "(Ljava/lang/Class;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)"
+                        + Type.getDescriptor(MethodRequest.class),
+                    false);
+                temp.visitInsn(Opcodes.DUP);
+                temp.visitVarInsn(Opcodes.ASTORE, requestLocal);
+                temp.visitMethodInsn(
+                    Opcodes.INVOKESTATIC,
+                    Type.getInternalName(AnnotationInstrumentationHelper.class),
+                    "enterCoroutine",
+                    "(ILkotlin/coroutines/Continuation;"
+                        + Type.getDescriptor(MethodRequest.class)
+                        + ")"
+                        + Type.getDescriptor(Context.class),
+                    false);
+                temp.visitInsn(Opcodes.DUP);
+                temp.visitVarInsn(Opcodes.ASTORE, contextLocal);
+                temp.visitMethodInsn(
+                    Opcodes.INVOKESTATIC,
+                    Type.getInternalName(AnnotationInstrumentationHelper.class),
+                    "openScope",
+                    "(" + Type.getDescriptor(Context.class) + ")" + Type.getDescriptor(Scope.class),
+                    false);
+                temp.visitVarInsn(Opcodes.ASTORE, scopeLocal);
+                // @SpanAttribute handling
+                for (Parameter parameter : annotatedParameters) {
+                  // label on stack, make a copy
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitLdcInsn(parameter.name);
+                  temp.visitVarInsn(parameter.type.getOpcode(Opcodes.ILOAD), parameter.var);
+                  boolean primitive =
+                      parameter.type.getSort() != Type.ARRAY
+                          && parameter.type.getSort() != Type.OBJECT;
+                  temp.visitMethodInsn(
+                      Opcodes.INVOKESTATIC,
+                      Type.getInternalName(AnnotationInstrumentationHelper.class),
+                      "setSpanAttribute",
+                      "(ILjava/lang/String;"
+                          + (primitive ? parameter.type.getDescriptor() : "Ljava/lang/Object;")
+                          + ")V",
+                      false);
+                }
+                // pop label
+                temp.visitInsn(Opcodes.POP);
+                if (hasBlockingOperation) {
+                  methodNode.instructions.insert(insertAfterInsn, temp.instructions);
+                } else {
+                  methodNode.instructions.insertBefore(
+                      methodNode.instructions.get(0), temp.instructions);
+                }
+              }
+
+              // insert at the start of the method
+              // null the local variables we added
+              // this is needed because jvm requires that a value needs to be assigned to the local
+              // before it is used, we need to initialize the locals that we use in the exception
+              // handler
+              // if the previous block was added at the start of the method this nulling step isn't
+              // necessary
+              if (hasBlockingOperation) {
+                MethodNode temp = new MethodNode();
+                temp.visitInsn(Opcodes.ACONST_NULL);
+                temp.visitVarInsn(Opcodes.ASTORE, requestLocal);
+                temp.visitInsn(Opcodes.ACONST_NULL);
+                temp.visitVarInsn(Opcodes.ASTORE, ourContinuationLocal);
+                temp.visitInsn(Opcodes.ACONST_NULL);
+                temp.visitVarInsn(Opcodes.ASTORE, contextLocal);
+                temp.visitInsn(Opcodes.ACONST_NULL);
+                temp.visitVarInsn(Opcodes.ASTORE, scopeLocal);
+
+                methodNode.instructions.insertBefore(
+                    methodNode.instructions.get(0), temp.instructions);
+              }
+
+              // insert exception handler code, this exception handler will catch Throwable
+              {
+                MethodNode temp = new MethodNode();
+                // lastLocal is the last local we added before the start of try block
+                int numLocals = lastLocal + 1;
+                Object[] locals = new Object[numLocals];
+                // in this handler we are using only the locals we added, we don't care about method
+                // arguments and this, so we don't list them in the stack frame
+                Arrays.fill(locals, Opcodes.TOP);
+                locals[requestLocal] = Type.getInternalName(MethodRequest.class);
+                locals[ourContinuationLocal] = Type.getInternalName(Continuation.class);
+                locals[contextLocal] = Type.getInternalName(Context.class);
+                locals[scopeLocal] = Type.getInternalName(Scope.class);
+
+                temp.visitFrame(
+                    Opcodes.F_NEW, numLocals, locals, 1, new Object[] {"java/lang/Throwable"});
+                // we have throwable on stack
+                // insert AnnotationInstrumentationHelper.exitCoroutine(exception, null, request,
+                // context, scope)
+                // that will close the scope and end span
+                temp.visitInsn(Opcodes.DUP);
+                temp.visitInsn(Opcodes.ACONST_NULL);
+                temp.visitVarInsn(Opcodes.ALOAD, requestLocal);
+                temp.visitVarInsn(Opcodes.ALOAD, ourContinuationLocal);
+                temp.visitVarInsn(Opcodes.ALOAD, contextLocal);
+                temp.visitVarInsn(Opcodes.ALOAD, scopeLocal);
+                temp.visitMethodInsn(
+                    Opcodes.INVOKESTATIC,
+                    Type.getInternalName(AnnotationInstrumentationHelper.class),
+                    "exitCoroutine",
+                    "(Ljava/lang/Throwable;Ljava/lang/Object;"
+                        + Type.getDescriptor(MethodRequest.class)
+                        + Type.getDescriptor(Continuation.class)
+                        + Type.getDescriptor(Context.class)
+                        + Type.getDescriptor(Scope.class)
+                        + ")V",
+                    false);
+
+                // rethrow the exception
+                temp.visitInsn(Opcodes.ATHROW);
+
+                methodNode.instructions.add(temp.instructions);
+              }
+
+              // insert code before each return instruction
+              // iterating instructions in reverse order to avoid having to deal with the
+              // instructions that we just added
+              for (int i = methodNode.instructions.size() - 1; i >= 0; i--) {
+                AbstractInsnNode instruction = methodNode.instructions.get(i);
+                // this method returns Object, so we don't need to handle other return instructions
+                if (instruction.getOpcode() == Opcodes.ARETURN) {
+                  MethodNode temp = new MethodNode();
+                  // we have return value on stack
+                  // insert AnnotationInstrumentationHelper.exitCoroutine(returnValue, request,
+                  // context, scope)
+                  // that will close the scope and end span if needed
+                  temp.visitInsn(Opcodes.DUP);
+                  temp.visitVarInsn(Opcodes.ALOAD, requestLocal);
+                  temp.visitVarInsn(Opcodes.ALOAD, ourContinuationLocal);
+                  temp.visitVarInsn(Opcodes.ALOAD, contextLocal);
+                  temp.visitVarInsn(Opcodes.ALOAD, scopeLocal);
+                  temp.visitMethodInsn(
+                      Opcodes.INVOKESTATIC,
+                      Type.getInternalName(AnnotationInstrumentationHelper.class),
+                      "exitCoroutine",
+                      "(Ljava/lang/Object;"
+                          + Type.getDescriptor(MethodRequest.class)
+                          + Type.getDescriptor(Continuation.class)
+                          + Type.getDescriptor(Context.class)
+                          + Type.getDescriptor(Scope.class)
+                          + ")V",
+                      false);
+                  methodNode.instructions.insertBefore(instruction, temp.instructions);
+                }
+              }
+
+              methodNode.accept(target);
+            }
+          };
+
+      return generatorAdapter;
+    }
+  }
+}

instrumentation/kotlinx-coroutines/javaagent/src/test/kotlin/io/opentelemetry/javaagent/instrumentation/kotlinxcoroutines/KotlinCoroutinesInstrumentationTest.kt

@@ -5,15 +5,21 @@
 
 package io.opentelemetry.javaagent.instrumentation.kotlinxcoroutines
 
+import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.api.trace.SpanKind
 import io.opentelemetry.context.Context
 import io.opentelemetry.context.ContextKey
 import io.opentelemetry.context.Scope
 import io.opentelemetry.extension.kotlin.asContextElement
 import io.opentelemetry.extension.kotlin.getOpenTelemetryContext
+import io.opentelemetry.instrumentation.annotations.SpanAttribute
+import io.opentelemetry.instrumentation.annotations.WithSpan
 import io.opentelemetry.instrumentation.reactor.v3_1.ContextPropagationOperator
 import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension
 import io.opentelemetry.instrumentation.testing.util.TelemetryDataUtil.orderByRootSpanName
+import io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo
 import io.opentelemetry.sdk.testing.assertj.TraceAssert
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.CoroutineScope
@@ -493,6 +499,66 @@ class KotlinCoroutinesInstrumentationTest {
     )
   }
 
+  @Test
+  fun `test WithSpan annotation`() {
+    runBlocking {
+      annotated1()
+    }
+
+    testing.waitAndAssertTraces(
+      { trace ->
+        trace.hasSpansSatisfyingExactly(
+          {
+            it.hasName("a1")
+              .hasNoParent()
+              .hasAttributesSatisfyingExactly(
+                equalTo(SemanticAttributes.CODE_NAMESPACE, this.javaClass.name),
+                equalTo(SemanticAttributes.CODE_FUNCTION, "annotated1")
+              )
+          },
+          {
+            it.hasName("KotlinCoroutinesInstrumentationTest.annotated2")
+              .hasParent(trace.getSpan(0))
+              .hasAttributesSatisfyingExactly(
+                equalTo(SemanticAttributes.CODE_NAMESPACE, this.javaClass.name),
+                equalTo(SemanticAttributes.CODE_FUNCTION, "annotated2"),
+                equalTo(AttributeKey.longKey("byteValue"), 1),
+                equalTo(AttributeKey.longKey("intValue"), 4),
+                equalTo(AttributeKey.longKey("longValue"), 5),
+                equalTo(AttributeKey.longKey("shortValue"), 6),
+                equalTo(AttributeKey.doubleKey("doubleValue"), 2.0),
+                equalTo(AttributeKey.doubleKey("floatValue"), 3.0),
+                equalTo(AttributeKey.booleanKey("booleanValue"), true),
+                equalTo(AttributeKey.stringKey("charValue"), "a"),
+                equalTo(AttributeKey.stringKey("stringValue"), "test")
+              )
+          }
+        )
+      }
+    )
+  }
+
+  @WithSpan(value = "a1", kind = SpanKind.CLIENT)
+  private suspend fun annotated1() {
+    delay(10)
+    annotated2(1, true, 'a', 2.0, 3.0f, 4, 5, 6, "test")
+  }
+
+  @WithSpan
+  private suspend fun annotated2(
+    @SpanAttribute byteValue: Byte,
+    @SpanAttribute booleanValue: Boolean,
+    @SpanAttribute charValue: Char,
+    @SpanAttribute doubleValue: Double,
+    @SpanAttribute floatValue: Float,
+    @SpanAttribute intValue: Int,
+    @SpanAttribute longValue: Long,
+    @SpanAttribute shortValue: Short,
+    @SpanAttribute("stringValue") s: String
+  ) {
+    delay(10)
+  }
+
   private fun tracedChild(opName: String) {
     tracer.spanBuilder(opName).startSpan().end()
   }

instrumentation/opentelemetry-instrumentation-annotations-1.16/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/instrumentationannotations/AnnotationExcludedMethods.java

@@ -18,7 +18,7 @@ import net.bytebuddy.description.method.MethodDescription;
 import net.bytebuddy.matcher.ElementMatcher;
 import net.bytebuddy.matcher.ElementMatchers;
 
-final class AnnotationExcludedMethods {
+public final class AnnotationExcludedMethods {
 
   private static final String TRACE_ANNOTATED_METHODS_EXCLUDE_CONFIG =
       "otel.instrumentation.opentelemetry-instrumentation-annotations.exclude-methods";
@@ -27,7 +27,7 @@ final class AnnotationExcludedMethods {
   Returns a matcher for all methods that should be excluded from auto-instrumentation by
   annotation-based advices.
   */
-  static ElementMatcher.Junction<MethodDescription> configureExcludedMethods() {
+  public static ElementMatcher.Junction<MethodDescription> configureExcludedMethods() {
     ElementMatcher.Junction<MethodDescription> result = none();
 
     Map<String, Set<String>> excludedMethods =

instrumentation/opentelemetry-instrumentation-annotations-1.16/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/instrumentationannotations/KotlinCoroutineUtil.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.instrumentationannotations;
+
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.method.ParameterList;
+import net.bytebuddy.matcher.ElementMatcher;
+
+public final class KotlinCoroutineUtil {
+
+  private KotlinCoroutineUtil() {}
+
+  public static ElementMatcher<MethodDescription> isKotlinSuspendMethod() {
+    // kotlin suspend methods return Object and take kotlin.coroutines.Continuation as last argument
+    return returns(Object.class)
+        .and(
+            target -> {
+              ParameterList<?> parameterList = target.getParameters();
+              if (!parameterList.isEmpty()) {
+                String lastParameter =
+                    parameterList.get(parameterList.size() - 1).getType().asErasure().getName();
+                return "kotlin.coroutines.Continuation".equals(lastParameter);
+              }
+              return false;
+            });
+  }
+}

instrumentation/opentelemetry-instrumentation-annotations-1.16/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/instrumentationannotations/WithSpanInstrumentation.java

@@ -7,6 +7,7 @@ package io.opentelemetry.javaagent.instrumentation.instrumentationannotations;
 
 import static io.opentelemetry.javaagent.instrumentation.instrumentationannotations.AnnotationSingletons.instrumenter;
 import static io.opentelemetry.javaagent.instrumentation.instrumentationannotations.AnnotationSingletons.instrumenterWithAttributes;
+import static io.opentelemetry.javaagent.instrumentation.instrumentationannotations.KotlinCoroutineUtil.isKotlinSuspendMethod;
 import static net.bytebuddy.matcher.ElementMatchers.declaresMethod;
 import static net.bytebuddy.matcher.ElementMatchers.hasParameters;
 import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
@@ -29,7 +30,7 @@ import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.implementation.bytecode.assign.Assigner;
 import net.bytebuddy.matcher.ElementMatcher;
 
-public class WithSpanInstrumentation implements TypeInstrumentation {
+class WithSpanInstrumentation implements TypeInstrumentation {
 
   private final ElementMatcher.Junction<AnnotationSource> annotatedMethodMatcher;
   private final ElementMatcher.Junction<MethodDescription> annotatedParametersMatcher;
@@ -45,7 +46,9 @@ public class WithSpanInstrumentation implements TypeInstrumentation {
                 isAnnotatedWith(
                     named(
                         "application.io.opentelemetry.instrumentation.annotations.SpanAttribute"))));
-    excludedMethodsMatcher = AnnotationExcludedMethods.configureExcludedMethods();
+    // exclude all kotlin suspend methods, these are handle in kotlinx-coroutines instrumentation
+    excludedMethodsMatcher =
+        AnnotationExcludedMethods.configureExcludedMethods().or(isKotlinSuspendMethod());
   }
 
   @Override

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/bytebuddy/ExceptionHandlers.java

@@ -75,6 +75,9 @@ public final class ExceptionHandlers {
               mv.visitLabel(handlerExit);
               if (frames) {
                 mv.visitFrame(Opcodes.F_SAME, 0, null, 0, null);
+                // there may be at most one frame at given code location, we need to add an extra
+                // NOP instruction to ensure that there isn't a duplicate frame
+                mv.visitInsn(Opcodes.NOP);
               }
 
               return size;

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/ignore/AdditionalLibraryIgnoredTypesConfigurer.java

@@ -271,6 +271,6 @@ public class AdditionalLibraryIgnoredTypesConfigurer implements IgnoredTypesConf
         .allowClass("com.fasterxml.jackson.databind.util.internal.PrivateMaxEntriesMap$AddTask");
 
     // kotlin, note we do not ignore kotlinx because we instrument coroutines code
-    builder.ignoreClass("kotlin.").allowClass("kotlin.coroutines.jvm.internal.DebugProbesKt");
+    builder.ignoreClass("kotlin.");
   }
 }