Add instrumentation for Quartz 2.0 (#4017)

* Add instrumentation for Quartz 2.0

* Fix drift in comment

* Fix fix

* Comment

* Cleanup

instrumentation/quartz-2.0/javaagent/build.gradle.kts

@@ -0,0 +1,20 @@
+plugins {
+  id("otel.javaagent-instrumentation")
+}
+
+muzzle {
+  pass {
+    group.set("org.quartz-scheduler")
+    module.set("quartz")
+    versions.set("[2.0.0,)")
+    assertInverse.set(true)
+  }
+}
+
+dependencies {
+  implementation(project(":instrumentation:quartz-2.0:library"))
+
+  library("org.quartz-scheduler:quartz:2.0.0")
+
+  testImplementation(project(":instrumentation:quartz-2.0:testing"))
+}

instrumentation/quartz-2.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/quartz/v2_0/QuartzIgnoredTypesConfigurer.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.quartz.v2_0;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.instrumentation.api.config.Config;
+import io.opentelemetry.javaagent.extension.ignore.IgnoredTypesBuilder;
+import io.opentelemetry.javaagent.extension.ignore.IgnoredTypesConfigurer;
+
+@AutoService(IgnoredTypesConfigurer.class)
+public class QuartzIgnoredTypesConfigurer implements IgnoredTypesConfigurer {
+
+  @Override
+  public void configure(Config config, IgnoredTypesBuilder builder) {
+    // Quartz executes jobs themselves in a synchronous way, there's no reason to propagate context
+    // between its scheduler threads.
+    builder.ignoreTaskClass("org.quartz");
+  }
+}

instrumentation/quartz-2.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/quartz/v2_0/QuartzInstrumentation.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.quartz.v2_0;
+
+import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.hasClassesNamed;
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isConstructor;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
+import io.opentelemetry.javaagent.instrumentation.api.CallDepth;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.quartz.Scheduler;
+
+public class QuartzInstrumentation implements TypeInstrumentation {
+  @Override
+  public ElementMatcher<ClassLoader> classLoaderOptimization() {
+    return hasClassesNamed("org.quartz.Scheduler");
+  }
+
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return hasSuperType(named("org.quartz.Scheduler"));
+  }
+
+  @Override
+  public void transform(TypeTransformer transformer) {
+    transformer.applyAdviceToMethod(
+        isConstructor(), this.getClass().getName() + "$ConstructorAdvice");
+  }
+
+  @SuppressWarnings("unused")
+  public static class ConstructorAdvice {
+
+    @Advice.OnMethodEnter(suppress = Throwable.class)
+    public static void trackCallDepth(@Advice.Local("otelCallDepth") CallDepth callDepth) {
+      callDepth = CallDepth.forClass(Scheduler.class);
+      callDepth.getAndIncrement();
+    }
+
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void addTracingInterceptor(
+        @Advice.This Scheduler scheduler, @Advice.Local("otelCallDepth") CallDepth callDepth) {
+      // No-args constructor is automatically called by constructors with args, but we only want to
+      // run once from the constructor with args because that is where the dedupe needs to happen.
+      if (callDepth.decrementAndGet() > 0) {
+        return;
+      }
+      QuartzSingletons.TRACING.configure(scheduler);
+    }
+  }
+}

instrumentation/quartz-2.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/quartz/v2_0/QuartzInstrumentationModule.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.quartz.v2_0;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import java.util.Collections;
+import java.util.List;
+
+@AutoService(InstrumentationModule.class)
+public class QuartzInstrumentationModule extends InstrumentationModule {
+
+  public QuartzInstrumentationModule() {
+    super("quartz", "quartz-2.0");
+  }
+
+  @Override
+  public List<TypeInstrumentation> typeInstrumentations() {
+    return Collections.singletonList(new QuartzInstrumentation());
+  }
+}

instrumentation/quartz-2.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/quartz/v2_0/QuartzSingletons.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.instrumentation.quartz.v2_0.QuartzTracing;
+
+public final class QuartzSingletons {
+
+  public static final QuartzTracing TRACING = QuartzTracing.create(GlobalOpenTelemetry.get());
+
+  private QuartzSingletons() {}
+}

instrumentation/quartz-2.0/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/quartz/v2_0/QuartzTest.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.instrumentation.quartz.v2_0.AbstractQuartzTest;
+import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.quartz.Scheduler;
+
+class QuartzTest extends AbstractQuartzTest {
+
+  @RegisterExtension InstrumentationExtension testing = AgentInstrumentationExtension.create();
+
+  @Override
+  protected void configureScheduler(Scheduler scheduler) {}
+
+  @Override
+  protected InstrumentationExtension getTesting() {
+    return testing;
+  }
+}

instrumentation/quartz-2.0/library/build.gradle.kts

@@ -0,0 +1,10 @@
+plugins {
+  id("otel.library-instrumentation")
+  id("otel.nullaway-conventions")
+}
+
+dependencies {
+  library("org.quartz-scheduler:quartz:2.0.0")
+
+  testImplementation(project(":instrumentation:quartz-2.0:testing"))
+}

instrumentation/quartz-2.0/library/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/QuartzErrorCauseExtractor.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.instrumentation.api.instrumenter.ErrorCauseExtractor;
+import org.quartz.SchedulerException;
+
+final class QuartzErrorCauseExtractor implements ErrorCauseExtractor {
+  @Override
+  public Throwable extractCause(Throwable error) {
+    Throwable userError = error;
+    while (userError instanceof SchedulerException) {
+      userError = ((SchedulerException) userError).getUnderlyingException();
+    }
+    return userError;
+  }
+}

instrumentation/quartz-2.0/library/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/QuartzSpanNameExtractor.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.instrumentation.api.instrumenter.SpanNameExtractor;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobKey;
+
+final class QuartzSpanNameExtractor implements SpanNameExtractor<JobExecutionContext> {
+  @Override
+  public String extract(JobExecutionContext job) {
+    JobKey key = job.getJobDetail().getKey();
+    return key.getGroup() + '.' + key.getName();
+  }
+}

instrumentation/quartz-2.0/library/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/QuartzTracing.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.api.OpenTelemetry;
+import org.quartz.JobListener;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.impl.matchers.EverythingMatcher;
+
+/** Entrypoint for tracing execution of Quartz jobs. */
+public final class QuartzTracing {
+
+  /** Returns a new {@link QuartzTracing} configured with the given {@link OpenTelemetry}. */
+  public static QuartzTracing create(OpenTelemetry openTelemetry) {
+    return builder(openTelemetry).build();
+  }
+
+  /** Returns a new {@link QuartzTracingBuilder} configured with the given {@link OpenTelemetry}. */
+  public static QuartzTracingBuilder builder(OpenTelemetry openTelemetry) {
+    return new QuartzTracingBuilder(openTelemetry);
+  }
+
+  QuartzTracing(JobListener jobListener) {
+    this.jobListener = jobListener;
+  }
+
+  private final JobListener jobListener;
+
+  /**
+   * Configures the {@link Scheduler} to enable tracing of jobs.
+   *
+   * <p><strong>NOTE:</strong> If there are job listeners already registered on the Scheduler that
+   * may throw exceptions, tracing will be broken. It's important to call this as soon as possible
+   * to avoid being affected by other bad listeners, or otherwise ensure listeners you register do
+   * not throw exceptions.
+   */
+  public void configure(Scheduler scheduler) {
+    try {
+      for (JobListener listener : scheduler.getListenerManager().getJobListeners()) {
+        if (listener instanceof TracingJobListener) {
+          return;
+        }
+      }
+    } catch (SchedulerException e) {
+      // Ignore
+    }
+    try {
+      // We must pass a matcher to work around a bug in Quartz 2.0.0. It's unlikely anyone uses
+      // a version before 2.0.2, but it makes muzzle simple.
+      scheduler.getListenerManager().addJobListener(jobListener, EverythingMatcher.allJobs());
+    } catch (SchedulerException e) {
+      throw new IllegalStateException("Could not add JobListener to Scheduler", e);
+    }
+  }
+}

instrumentation/quartz-2.0/library/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/QuartzTracingBuilder.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
+import java.util.ArrayList;
+import java.util.List;
+import org.quartz.JobExecutionContext;
+
+/** A builder of {@link QuartzTracing}. */
+public final class QuartzTracingBuilder {
+
+  private static final String INSTRUMENTATION_NAME = "io.opentelemetry.quartz-1.7";
+
+  private final OpenTelemetry openTelemetry;
+
+  private final List<AttributesExtractor<? super JobExecutionContext, ? super Void>>
+      additionalExtractors = new ArrayList<>();
+
+  QuartzTracingBuilder(OpenTelemetry openTelemetry) {
+    this.openTelemetry = openTelemetry;
+  }
+
+  /**
+   * Adds an additional {@link AttributesExtractor} to invoke to set attributes to instrumented
+   * items. The {@link AttributesExtractor} will be executed after all default extractors.
+   */
+  public QuartzTracingBuilder addAttributeExtractor(
+      AttributesExtractor<? super JobExecutionContext, ? super Void> attributesExtractor) {
+    additionalExtractors.add(attributesExtractor);
+    return this;
+  }
+
+  /** Returns a new {@link QuartzTracing} with the settings of this {@link QuartzTracingBuilder}. */
+  public QuartzTracing build() {
+    InstrumenterBuilder<JobExecutionContext, Void> instrumenter =
+        Instrumenter.newBuilder(openTelemetry, INSTRUMENTATION_NAME, new QuartzSpanNameExtractor());
+
+    instrumenter.setErrorCauseExtractor(new QuartzErrorCauseExtractor());
+    instrumenter.addAttributesExtractors(additionalExtractors);
+
+    return new QuartzTracing(new TracingJobListener(instrumenter.newInstrumenter()));
+  }
+}

instrumentation/quartz-2.0/library/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/TracingJobListener.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.quartz.JobListener;
+
+final class TracingJobListener implements JobListener {
+
+  private final Instrumenter<JobExecutionContext, Void> instrumenter;
+
+  TracingJobListener(Instrumenter<JobExecutionContext, Void> instrumenter) {
+    this.instrumenter = instrumenter;
+  }
+
+  @Override
+  public String getName() {
+    return TracingJobListener.class.getName();
+  }
+
+  @Override
+  public void jobExecutionVetoed(JobExecutionContext jobExecutionContext) {
+    // TODO(anuraaga): Consider adding an attribute for vetoed jobs.
+  }
+
+  @Override
+  public void jobToBeExecuted(JobExecutionContext job) {
+    Context parentCtx = Context.current();
+    if (!instrumenter.shouldStart(parentCtx, job)) {
+      return;
+    }
+
+    Context context = instrumenter.start(parentCtx, job);
+    job.put(Context.class, context);
+
+    // Listeners are executed synchronously on the same thread starting here.
+    // https://github.com/quartz-scheduler/quartz/blob/quartz-2.0.x/quartz/src/main/java/org/quartz/core/JobRunShell.java#L180
+    // However, if a listener before this one throws an exception in wasExecuted, we won't be
+    // executed. Library instrumentation users need to make sure other listeners don't throw
+    // exceptions.
+    Scope scope = context.makeCurrent();
+    job.put(Scope.class, scope);
+  }
+
+  @Override
+  public void jobWasExecuted(JobExecutionContext job, JobExecutionException error) {
+    Scope scope = (Scope) job.get(Scope.class);
+    if (scope != null) {
+      scope.close();
+    }
+
+    Context context = (Context) job.get(Context.class);
+    if (context == null) {
+      // Would only happen if we didn't start a span (maybe a previous joblistener threw an
+      // exception before ours could process the start event).
+      return;
+    }
+
+    instrumenter.end(context, job, null, error);
+  }
+}

instrumentation/quartz-2.0/library/src/test/java/io/opentelemetry/instrumentation/quartz/v2_0/QuartzTest.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.quartz.Scheduler;
+
+class QuartzTest extends AbstractQuartzTest {
+
+  @RegisterExtension InstrumentationExtension testing = LibraryInstrumentationExtension.create();
+
+  @Override
+  protected void configureScheduler(Scheduler scheduler) {
+    QuartzTracing.create(testing.getOpenTelemetry()).configure(scheduler);
+  }
+
+  @Override
+  protected InstrumentationExtension getTesting() {
+    return testing;
+  }
+}

instrumentation/quartz-2.0/testing/build.gradle.kts

@@ -0,0 +1,9 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+dependencies {
+  api(project(":testing-common"))
+
+  api("org.quartz-scheduler:quartz:2.0.0")
+}

instrumentation/quartz-2.0/testing/src/main/java/io/opentelemetry/instrumentation/quartz/v2_0/AbstractQuartzTest.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.quartz.v2_0;
+
+import static org.quartz.JobBuilder.newJob;
+import static org.quartz.TriggerBuilder.newTrigger;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.sdk.trace.data.StatusData;
+import java.util.Properties;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.quartz.Job;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.Scheduler;
+import org.quartz.Trigger;
+import org.quartz.impl.StdSchedulerFactory;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class AbstractQuartzTest {
+
+  protected abstract void configureScheduler(Scheduler scheduler);
+
+  private Scheduler scheduler;
+
+  protected abstract InstrumentationExtension getTesting();
+
+  @BeforeAll
+  void startScheduler() throws Exception {
+    scheduler = createScheduler("default");
+    configureScheduler(scheduler);
+    scheduler.start();
+  }
+
+  @AfterAll
+  void stopScheduler() throws Exception {
+    scheduler.shutdown();
+  }
+
+  @Test
+  void successfulJob() throws Exception {
+    Trigger trigger = newTrigger().build();
+
+    JobDetail jobDetail = newJob().withIdentity("test", "jobs").ofType(SuccessfulJob.class).build();
+
+    scheduler.scheduleJob(jobDetail, trigger);
+
+    getTesting()
+        .waitAndAssertTraces(
+            trace ->
+                trace.hasSpansSatisfyingExactly(
+                    span ->
+                        span.hasName("jobs.test")
+                            .hasKind(SpanKind.INTERNAL)
+                            .hasNoParent()
+                            .hasStatus(StatusData.unset())
+                            .hasAttributes(Attributes.empty()),
+                    span ->
+                        span.hasName("child")
+                            .hasKind(SpanKind.INTERNAL)
+                            .hasParent(trace.getSpan(0))));
+  }
+
+  @Test
+  void failingJob() throws Exception {
+    Trigger trigger = newTrigger().build();
+
+    JobDetail jobDetail = newJob().withIdentity("fail", "jobs").ofType(FailingJob.class).build();
+
+    scheduler.scheduleJob(jobDetail, trigger);
+
+    getTesting()
+        .waitAndAssertTraces(
+            trace ->
+                trace.hasSpansSatisfyingExactly(
+                    span ->
+                        span.hasName("jobs.fail")
+                            .hasKind(SpanKind.INTERNAL)
+                            .hasNoParent()
+                            .hasStatus(StatusData.error())
+                            .hasException(new IllegalStateException("Bad job"))
+                            .hasAttributes(Attributes.empty())));
+  }
+
+  private static Scheduler createScheduler(String name) throws Exception {
+    StdSchedulerFactory factory = new StdSchedulerFactory();
+    Properties properties = new Properties();
+    properties.load(AbstractQuartzTest.class.getResourceAsStream("/org/quartz/quartz.properties"));
+    properties.put(StdSchedulerFactory.PROP_SCHED_INSTANCE_NAME, name);
+    factory.initialize(properties);
+    return factory.getScheduler();
+  }
+
+  public static class SuccessfulJob implements Job {
+    @Override
+    public void execute(JobExecutionContext context) {
+      GlobalOpenTelemetry.getTracer("jobtracer").spanBuilder("child").startSpan().end();
+    }
+  }
+
+  public static class FailingJob implements Job {
+    @Override
+    public void execute(JobExecutionContext context) {
+      throw new IllegalStateException("Bad job");
+    }
+  }
+}

settings.gradle.kts

@@ -264,6 +264,9 @@ include(":instrumentation:play-ws:play-ws-2.0:javaagent")
 include(":instrumentation:play-ws:play-ws-2.1:javaagent")
 include(":instrumentation:play-ws:play-ws-common:javaagent")
 include(":instrumentation:play-ws:play-ws-testing")
+include(":instrumentation:quartz-2.0:javaagent")
+include(":instrumentation:quartz-2.0:library")
+include(":instrumentation:quartz-2.0:testing")
 include(":instrumentation:rabbitmq-2.7:javaagent")
 include(":instrumentation:ratpack-1.4:javaagent")
 include(":instrumentation:ratpack-1.4:library")