diff --git a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index b46802701c6..c46a2b2781b 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -249,7 +249,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat settings.disabledTests, settings.attemptToFixTests, settings.diff, - false) + ConfigurationErrors.NONE) } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java index 2a6344e5ed6..d7502ccec48 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java @@ -24,7 +24,7 @@ public class CiVisibilitySettings { null, false); - public static final CiVisibilitySettings REQUEST_ERROR = + public static final CiVisibilitySettings SETTINGS_REQUEST_ERROR = new CiVisibilitySettings( false, false, @@ -52,7 +52,7 @@ public class CiVisibilitySettings { private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings; private final TestManagementSettings testManagementSettings; @Nullable private final String defaultBranch; - private final boolean requestError; + private final boolean settingsRequestError; CiVisibilitySettings( boolean itrEnabled, @@ -67,7 +67,7 @@ public class CiVisibilitySettings { EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, TestManagementSettings testManagementSettings, @Nullable String defaultBranch, - boolean requestError) { + boolean settingsRequestError) { this.itrEnabled = itrEnabled; this.codeCoverage = codeCoverage; this.testsSkipping = testsSkipping; @@ -80,7 +80,7 @@ public class CiVisibilitySettings { this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings; this.testManagementSettings = testManagementSettings; this.defaultBranch = defaultBranch; - this.requestError = requestError; + this.settingsRequestError = settingsRequestError; } public boolean isItrEnabled() { @@ -132,8 +132,8 @@ public String getDefaultBranch() { return defaultBranch; } - public boolean isRequestError() { - return requestError; + public boolean isSettingsRequestError() { + return settingsRequestError; } @Override @@ -157,7 +157,7 @@ public boolean equals(Object o) { && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings) && Objects.equals(testManagementSettings, that.testManagementSettings) && Objects.equals(defaultBranch, that.defaultBranch) - && requestError == that.requestError; + && settingsRequestError == that.settingsRequestError; } @Override @@ -175,7 +175,7 @@ public int hashCode() { earlyFlakeDetectionSettings, testManagementSettings, defaultBranch, - requestError); + settingsRequestError); } public interface Factory { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationErrors.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationErrors.java new file mode 100644 index 00000000000..f488468f2f4 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationErrors.java @@ -0,0 +1,106 @@ +package datadog.trace.civisibility.config; + +import datadog.trace.api.DDTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.civisibility.ipc.serialization.Serializer; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** Tracks which CI Visibility backend requests failed. */ +public class ConfigurationErrors { + + public static final ConfigurationErrors NONE = + new ConfigurationErrors(false, false, false, false, false); + + private static final int SETTINGS_FLAG = 1; + private static final int SKIPPABLE_TESTS_FLAG = 2; + private static final int FLAKY_TESTS_FLAG = 4; + private static final int KNOWN_TESTS_FLAG = 8; + private static final int TEST_MANAGEMENT_TESTS_FLAG = 16; + + private final boolean settings; + private final boolean skippableTests; + private final boolean flakyTests; + private final boolean knownTests; + private final boolean testManagementTests; + + public ConfigurationErrors( + boolean settings, + boolean skippableTests, + boolean flakyTests, + boolean knownTests, + boolean testManagementTests) { + this.settings = settings; + this.skippableTests = skippableTests; + this.flakyTests = flakyTests; + this.knownTests = knownTests; + this.testManagementTests = testManagementTests; + } + + public boolean hasAny() { + return settings || skippableTests || flakyTests || knownTests || testManagementTests; + } + + public void applyTags(AgentSpan span) { + if (settings) { + span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SETTINGS, true); + } + if (skippableTests) { + span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SKIPPABLE_TESTS, true); + } + if (flakyTests) { + span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_FLAKY_TESTS, true); + } + if (knownTests) { + span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_KNOWN_TESTS, true); + } + if (testManagementTests) { + span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_TEST_MANAGEMENT_TESTS, true); + } + } + + public static void serialize(Serializer s, ConfigurationErrors errors) { + byte flags = + (byte) + ((errors.settings ? SETTINGS_FLAG : 0) + | (errors.skippableTests ? SKIPPABLE_TESTS_FLAG : 0) + | (errors.flakyTests ? FLAKY_TESTS_FLAG : 0) + | (errors.knownTests ? KNOWN_TESTS_FLAG : 0) + | (errors.testManagementTests ? TEST_MANAGEMENT_TESTS_FLAG : 0)); + s.write(flags); + } + + public static ConfigurationErrors deserialize(ByteBuffer buffer) { + byte flags = Serializer.readByte(buffer); + if (flags == 0) { + return NONE; + } + return new ConfigurationErrors( + (flags & SETTINGS_FLAG) != 0, + (flags & SKIPPABLE_TESTS_FLAG) != 0, + (flags & FLAKY_TESTS_FLAG) != 0, + (flags & KNOWN_TESTS_FLAG) != 0, + (flags & TEST_MANAGEMENT_TESTS_FLAG) != 0); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConfigurationErrors that = (ConfigurationErrors) o; + return settings == that.settings + && skippableTests == that.skippableTests + && flakyTests == that.flakyTests + && knownTests == that.knownTests + && testManagementTests == that.testManagementTests; + } + + @Override + public int hashCode() { + return Objects.hash(settings, skippableTests, flakyTests, knownTests, testManagementTests); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java index f506ec659b4..36e48401132 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java @@ -39,9 +39,9 @@ public class ExecutionSettings { Collections.emptyList(), Collections.emptyList(), LineDiff.EMPTY, - false); + ConfigurationErrors.NONE); - public static final ExecutionSettings REQUEST_ERROR = + public static final ExecutionSettings SETTINGS_REQUEST_ERROR = new ExecutionSettings( false, false, @@ -61,7 +61,7 @@ public class ExecutionSettings { Collections.emptyList(), Collections.emptyList(), LineDiff.EMPTY, - true); + new ConfigurationErrors(true, false, false, false, false)); private final boolean itrEnabled; private final boolean codeCoverageEnabled; @@ -78,7 +78,7 @@ public class ExecutionSettings { @Nonnull private final Map testSettings; @Nonnull private final Map settingsCount; @Nonnull private final Diff pullRequestDiff; - private final boolean configurationError; + @Nonnull private final ConfigurationErrors configurationErrors; public ExecutionSettings( boolean itrEnabled, @@ -99,7 +99,7 @@ public ExecutionSettings( @Nonnull Collection disabledTests, @Nonnull Collection attemptToFixTests, @Nonnull Diff pullRequestDiff, - boolean configurationError) { + @Nonnull ConfigurationErrors configurationErrors) { this.itrEnabled = itrEnabled; this.codeCoverageEnabled = codeCoverageEnabled; this.testSkippingEnabled = testSkippingEnabled; @@ -113,7 +113,7 @@ public ExecutionSettings( this.skippableTests = skippableTests; this.skippableTestsCoverage = skippableTestsCoverage; this.pullRequestDiff = pullRequestDiff; - this.configurationError = configurationError; + this.configurationErrors = configurationErrors; testSettings = new HashMap<>(); if (flakyTests != null) { @@ -154,7 +154,7 @@ private ExecutionSettings( @Nonnull Map testSettings, @Nonnull EnumMap settingsCount, @Nonnull Diff pullRequestDiff, - boolean configurationError) { + @Nonnull ConfigurationErrors configurationErrors) { this.itrEnabled = itrEnabled; this.codeCoverageEnabled = codeCoverageEnabled; this.testSkippingEnabled = testSkippingEnabled; @@ -170,7 +170,7 @@ private ExecutionSettings( this.testSettings = testSettings; this.settingsCount = settingsCount; this.pullRequestDiff = pullRequestDiff; - this.configurationError = configurationError; + this.configurationErrors = configurationErrors; } /** @@ -277,8 +277,9 @@ public Diff getPullRequestDiff() { return pullRequestDiff; } - public boolean isConfigurationError() { - return configurationError; + @Nonnull + public ConfigurationErrors getConfigurationErrors() { + return configurationErrors; } @Override @@ -305,7 +306,7 @@ public boolean equals(Object o) { && Objects.equals(testSettings, that.testSettings) && Objects.equals(settingsCount, that.settingsCount) && Objects.equals(pullRequestDiff, that.pullRequestDiff) - && configurationError == that.configurationError; + && Objects.equals(configurationErrors, that.configurationErrors); } @Override @@ -326,7 +327,7 @@ public int hashCode() { testSettings, settingsCount, pullRequestDiff, - configurationError); + configurationErrors); } public static class Serializer { @@ -338,7 +339,6 @@ public static class Serializer { private static final int IMPACTED_TESTS_DETECTION_ENABLED_FLAG = 16; private static final int CODE_COVERAGE_REPORT_UPLOAD_ENABLED_FLAG = 32; private static final int FAILED_TEST_REPLAY_ENABLED_FLAG = 64; - private static final int CONFIGURATION_ERROR_FLAG = 128; public static ByteBuffer serialize(ExecutionSettings settings) { datadog.trace.civisibility.ipc.serialization.Serializer s = @@ -356,10 +356,11 @@ public static ByteBuffer serialize(ExecutionSettings settings) { | (settings.codeCoverageReportUploadEnabled ? CODE_COVERAGE_REPORT_UPLOAD_ENABLED_FLAG : 0) - | (settings.failedTestReplayEnabled ? FAILED_TEST_REPLAY_ENABLED_FLAG : 0) - | (settings.configurationError ? CONFIGURATION_ERROR_FLAG : 0)); + | (settings.failedTestReplayEnabled ? FAILED_TEST_REPLAY_ENABLED_FLAG : 0)); s.write(flags); + ConfigurationErrors.serialize(s, settings.configurationErrors); + EarlyFlakeDetectionSettings.Serializer.serialize(s, settings.earlyFlakeDetectionSettings); TestManagementSettings.Serializer.serialize(s, settings.testManagementSettings); @@ -399,7 +400,8 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { boolean codeCoverageReportUploadEnabled = (flags & CODE_COVERAGE_REPORT_UPLOAD_ENABLED_FLAG) != 0; boolean failedTestReplayEnabled = (flags & FAILED_TEST_REPLAY_ENABLED_FLAG) != 0; - boolean configurationError = (flags & CONFIGURATION_ERROR_FLAG) != 0; + + ConfigurationErrors configurationErrors = ConfigurationErrors.deserialize(buffer); EarlyFlakeDetectionSettings earlyFlakeDetectionSettings = EarlyFlakeDetectionSettings.Serializer.deserialize(buffer); @@ -452,7 +454,7 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { testSettings, settingsCount, diff, - configurationError); + configurationErrors); } } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java index fc68956aeaa..46ecb133ff3 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java @@ -27,6 +27,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -131,11 +132,11 @@ private Map create(TracerEnvironment tracerEnvironmen } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.error("Interrupted while creating execution settings"); - return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.REQUEST_ERROR); + return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.SETTINGS_REQUEST_ERROR); } catch (ExecutionException e) { LOGGER.error("Error while creating execution settings", e); - return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.REQUEST_ERROR); + return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.SETTINGS_REQUEST_ERROR); } finally { settingsExecutor.shutdownNow(); @@ -146,8 +147,8 @@ private Map create(TracerEnvironment tracerEnvironmen private Map doCreate( TracerEnvironment tracerEnvironment, CiVisibilitySettings settings, ExecutorService executor) throws InterruptedException, ExecutionException { - if (settings.isRequestError()) { - return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.REQUEST_ERROR); + if (settings.isSettingsRequestError()) { + return Collections.singletonMap(DEFAULT_SETTINGS, ExecutionSettings.SETTINGS_REQUEST_ERROR); } boolean itrEnabled = @@ -223,17 +224,28 @@ private Map doCreate( codeCoverageReportUpload, failedTestReplayEnabled); + AtomicBoolean skippableTestsError = new AtomicBoolean(); + AtomicBoolean flakyTestsError = new AtomicBoolean(); + AtomicBoolean knownTestsError = new AtomicBoolean(); + AtomicBoolean testManagementTestsError = new AtomicBoolean(); + Future skippableTestsFuture = - executor.submit(() -> getSkippableTests(tracerEnvironment, itrEnabled)); + executor.submit( + () -> getSkippableTests(tracerEnvironment, itrEnabled, skippableTestsError)); Future>> flakyTestsFuture = - executor.submit(() -> getFlakyTestsByModule(tracerEnvironment, flakyTestRetriesEnabled)); + executor.submit( + () -> + getFlakyTestsByModule(tracerEnvironment, flakyTestRetriesEnabled, flakyTestsError)); Future>> knownTestsFuture = - executor.submit(() -> getKnownTestsByModule(tracerEnvironment, knownTestsRequest)); + executor.submit( + () -> getKnownTestsByModule(tracerEnvironment, knownTestsRequest, knownTestsError)); Future>>> testManagementTestsFuture = executor.submit( () -> getTestManagementTestsByModule( - tracerEnvironment, testManagementSettings.isEnabled())); + tracerEnvironment, + testManagementSettings.isEnabled(), + testManagementTestsError)); Future pullRequestDiffFuture = executor.submit( () -> getPullRequestDiff(impactedTestsEnabled, settings.getDefaultBranch())); @@ -254,6 +266,14 @@ private Map doCreate( Diff pullRequestDiff = pullRequestDiffFuture.get(); + ConfigurationErrors configurationErrors = + new ConfigurationErrors( + false, + skippableTestsError.get(), + flakyTestsError.get(), + knownTestsError.get(), + testManagementTestsError.get()); + Map settingsByModule = new HashMap<>(); Set moduleNames = getModuleNames( @@ -292,7 +312,7 @@ private Map doCreate( disabledTestsByModule.getOrDefault(moduleName, Collections.emptyList()), attemptToFixTestsByModule.getOrDefault(moduleName, Collections.emptyList()), pullRequestDiff, - false)); + configurationErrors)); } return settingsByModule; } @@ -313,7 +333,7 @@ private CiVisibilitySettings getCiVisibilitySettings(TracerEnvironment tracerEnv } catch (Exception e) { LOGGER.error("Error while obtaining CI Visibility settings", e); - return CiVisibilitySettings.REQUEST_ERROR; + return CiVisibilitySettings.SETTINGS_REQUEST_ERROR; } } @@ -346,7 +366,7 @@ private TestManagementSettings getTestManagementSettings(CiVisibilitySettings se @Nonnull private SkippableTests getSkippableTests( - TracerEnvironment tracerEnvironment, boolean itrEnabled) { + TracerEnvironment tracerEnvironment, boolean itrEnabled, AtomicBoolean errorFlag) { if (!itrEnabled || repositoryRoot == null) { return SkippableTests.EMPTY; } @@ -375,16 +395,20 @@ private SkippableTests getSkippableTests( } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.error("Interrupted while waiting for git data upload", e); + errorFlag.set(true); return SkippableTests.EMPTY; } catch (Exception e) { LOGGER.error("Could not obtain list of skippable tests, will proceed without skipping", e); + errorFlag.set(true); return SkippableTests.EMPTY; } } @Nullable private Map> getFlakyTestsByModule( - TracerEnvironment tracerEnvironment, boolean flakyTestRetriesEnabled) { + TracerEnvironment tracerEnvironment, + boolean flakyTestRetriesEnabled, + AtomicBoolean errorFlag) { if (!(flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()) && !CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())) { return null; @@ -393,13 +417,14 @@ private Map> getFlakyTestsByModule( return configurationApi.getFlakyTestsByModule(tracerEnvironment); } catch (Exception e) { LOGGER.error("Could not obtain list of flaky tests", e); + errorFlag.set(true); return null; } } @Nullable private Map> getKnownTestsByModule( - TracerEnvironment tracerEnvironment, boolean knownTestsRequest) { + TracerEnvironment tracerEnvironment, boolean knownTestsRequest, AtomicBoolean errorFlag) { if (!knownTestsRequest) { return null; } @@ -408,13 +433,16 @@ private Map> getKnownTestsByModule( } catch (Exception e) { LOGGER.error("Could not obtain list of known tests", e); + errorFlag.set(true); return null; } } @Nullable private Map>> getTestManagementTestsByModule( - TracerEnvironment tracerEnvironment, boolean testManagementTestsRequest) { + TracerEnvironment tracerEnvironment, + boolean testManagementTestsRequest, + AtomicBoolean errorFlag) { if (!testManagementTestsRequest) { return Collections.emptyMap(); } @@ -431,6 +459,7 @@ private Map>> getTestManagementTest } } catch (Exception e) { LOGGER.error("Could not obtain list of test management tests", e); + errorFlag.set(true); return Collections.emptyMap(); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java index c21e793f039..19a4650ea79 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java @@ -5,7 +5,6 @@ import static datadog.trace.civisibility.Constants.CI_VISIBILITY_INSTRUMENTATION_NAME; import datadog.trace.api.Config; -import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.DDTest; @@ -41,6 +40,7 @@ import datadog.trace.bootstrap.instrumentation.api.TagContext; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.codeowners.Codeowners; +import datadog.trace.civisibility.config.ConfigurationErrors; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.source.LinesResolver; import datadog.trace.civisibility.source.SourcePathResolver; @@ -91,7 +91,7 @@ public TestImpl( Codeowners codeowners, CoverageStore.Factory coverageStoreFactory, ExecutionResults executionResults, - boolean configurationError, + @Nonnull ConfigurationErrors configurationErrors, @Nonnull Collection capabilities, Consumer onSpanFinish) { this.instrumentation = instrumentation; @@ -159,9 +159,7 @@ public TestImpl( testDecorator.afterStart(span); - if (configurationError) { - span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, true); - } + configurationErrors.applyTags(span); metricCollector.add(CiVisibilityCountMetric.EVENT_CREATED, 1, instrumentation, EventType.TEST); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java index 5e9df3f58b9..2b09f0cd0a7 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java @@ -5,7 +5,6 @@ import static datadog.trace.civisibility.Constants.CI_VISIBILITY_INSTRUMENTATION_NAME; import datadog.trace.api.Config; -import datadog.trace.api.DDTags; import datadog.trace.api.civisibility.DDTestSuite; import datadog.trace.api.civisibility.config.LibraryCapability; import datadog.trace.api.civisibility.coverage.CoverageStore; @@ -20,6 +19,7 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.codeowners.Codeowners; +import datadog.trace.civisibility.config.ConfigurationErrors; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.source.LinesResolver; import datadog.trace.civisibility.source.SourcePathResolver; @@ -54,7 +54,7 @@ public class TestSuiteImpl implements DDTestSuite { private final CoverageStore.Factory coverageStoreFactory; private final ExecutionResults executionResults; private final boolean parallelized; - private final boolean configurationError; + private final ConfigurationErrors configurationErrors; private final Collection capabilities; private final Consumer onSpanFinish; private final SpanTagsPropagator tagsPropagator; @@ -77,7 +77,7 @@ public TestSuiteImpl( LinesResolver linesResolver, CoverageStore.Factory coverageStoreFactory, ExecutionResults executionResults, - boolean configurationError, + @Nonnull ConfigurationErrors configurationErrors, @Nonnull Collection capabilities, Consumer onSpanFinish) { this.moduleSpanContext = moduleSpanContext; @@ -95,7 +95,7 @@ public TestSuiteImpl( this.linesResolver = linesResolver; this.coverageStoreFactory = coverageStoreFactory; this.executionResults = executionResults; - this.configurationError = configurationError; + this.configurationErrors = configurationErrors; this.capabilities = capabilities; this.onSpanFinish = onSpanFinish; @@ -135,9 +135,7 @@ public TestSuiteImpl( testDecorator.afterStart(span); - if (configurationError) { - span.setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, true); - } + configurationErrors.applyTags(span); if (!parallelized) { activateSpanWithoutScope(span); @@ -272,7 +270,7 @@ public TestImpl testStart( codeowners, coverageStoreFactory, executionResults, - configurationError, + configurationErrors, capabilities, tagsPropagator::propagateStatus); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java index 0c0e9ccddeb..48915cdb479 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java @@ -111,9 +111,7 @@ public BuildSystemModuleImpl( setTag(Tags.TEST_COMMAND, startCommand); - if (executionSettings.isConfigurationError()) { - setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, true); - } + executionSettings.getConfigurationErrors().applyTags(span); } @ParametersAreNonnullByDefault diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java index 0acb65459c9..de116acf05c 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java @@ -195,7 +195,12 @@ private void onModuleFinish(AgentSpan moduleSpan) { TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED, Boolean::logicalOr), TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED, Boolean::logicalOr), TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY, Boolean::logicalOr), - TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, Boolean::logicalOr)); + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SETTINGS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SKIPPABLE_TESTS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_FLAKY_TESTS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_KNOWN_TESTS, Boolean::logicalOr), + TagMergeSpec.of( + DDTags.CI_LIBRARY_CONFIGURATION_ERROR_TEST_MANAGEMENT_TESTS, Boolean::logicalOr)); } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java index e9653dc0209..43c4e8748cb 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java @@ -212,7 +212,7 @@ public TestSuiteImpl testSuiteStart( linesResolver, coverageStoreFactory, executionResults, - executionStrategy.getExecutionSettings().isConfigurationError(), + executionStrategy.getExecutionSettings().getConfigurationErrors(), capabilities, this::propagateTestFrameworkData); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java index af38cb09a90..142c2abd46b 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java @@ -124,9 +124,7 @@ public int executionPriority(@Nullable TestIdentifier test, @Nonnull TestSourceD @Override public void end(@Nullable Long endTime) { ExecutionSettings executionSettings = executionStrategy.getExecutionSettings(); - if (executionSettings.isConfigurationError()) { - setTag(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, true); - } + executionSettings.getConfigurationErrors().applyTags(span); if (executionSettings.isCodeCoverageEnabled()) { setTag(Tags.TEST_CODE_COVERAGE_ENABLED, true); @@ -189,7 +187,7 @@ public TestSuiteImpl testSuiteStart( linesResolver, coverageStoreFactory, executionResults, - executionStrategy.getExecutionSettings().isConfigurationError(), + executionStrategy.getExecutionSettings().getConfigurationErrors(), capabilities, tagsPropagator::propagateCiVisibilityTags); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestSession.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestSession.java index d1c7314374c..1024104361c 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestSession.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestSession.java @@ -95,7 +95,12 @@ private void onModuleFinish(AgentSpan moduleSpan) { TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED), TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED), TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY), - TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR, Boolean::logicalOr)); + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SETTINGS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_SKIPPABLE_TESTS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_FLAKY_TESTS, Boolean::logicalOr), + TagMergeSpec.of(DDTags.CI_LIBRARY_CONFIGURATION_ERROR_KNOWN_TESTS, Boolean::logicalOr), + TagMergeSpec.of( + DDTags.CI_LIBRARY_CONFIGURATION_ERROR_TEST_MANAGEMENT_TESTS, Boolean::logicalOr)); } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java index a00c45a8fbc..3ec4196401f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java @@ -9,6 +9,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.codeowners.Codeowners; +import datadog.trace.civisibility.config.ConfigurationErrors; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.domain.AbstractTestModule; import datadog.trace.civisibility.domain.InstrumentationType; @@ -81,7 +82,7 @@ public ManualApiTestSuite testSuiteStart( linesResolver, coverageStoreFactory, executionResults, - false, + ConfigurationErrors.NONE, Collections.emptyList(), tagsPropagator::propagateCiVisibilityTags); diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy index 082c0f60c87..e98a9bcadc6 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy @@ -39,7 +39,7 @@ class ExecutionSettingsTest extends DDSpecification { new HashSet<>([]), new HashSet<>([]), LineDiff.EMPTY, - false), + ConfigurationErrors.NONE), new ExecutionSettings( true, @@ -60,7 +60,7 @@ class ExecutionSettingsTest extends DDSpecification { new HashSet<>([new TestFQN("suite", "disabled")]), new HashSet<>([new TestFQN("suite", "attemptToFix")]), new LineDiff(["path": lines()]), - false + ConfigurationErrors.NONE ), new ExecutionSettings( @@ -86,7 +86,7 @@ class ExecutionSettingsTest extends DDSpecification { new HashSet<>([new TestFQN("suite", "disabled"), new TestFQN("another", "another-disabled")]), new HashSet<>([new TestFQN("suite", "attemptToFix"), new TestFQN("another", "another-attemptToFix")]), new LineDiff(["path": lines(1, 2, 3)]), - false + new ConfigurationErrors(false, true, false, true, false) ), new ExecutionSettings( @@ -112,7 +112,7 @@ class ExecutionSettingsTest extends DDSpecification { new HashSet<>([new TestFQN("suite", "disabled"), new TestFQN("another", "another-disabled")]), new HashSet<>([new TestFQN("suite", "attemptToFix"), new TestFQN("another", "another-attemptToFix")]), new LineDiff(["path": lines(1, 2, 3), "path-b": lines(1, 2, 128, 257, 999)]), - true + new ConfigurationErrors(true, true, true, true, true) ), ] } diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestImplTest.groovy index 0e32ecc24ba..5c1d7801e55 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestImplTest.groovy @@ -13,6 +13,7 @@ import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext import datadog.trace.bootstrap.instrumentation.api.AgentTracer import datadog.trace.civisibility.codeowners.NoCodeowners +import datadog.trace.civisibility.config.ConfigurationErrors import datadog.trace.civisibility.decorator.TestDecoratorImpl import datadog.trace.civisibility.source.LinesResolver import datadog.trace.civisibility.source.NoOpSourcePathResolver @@ -141,7 +142,7 @@ class TestImplTest extends SpanWriterTest { codeowners, coverageStoreFactory, executionResults, - false, + ConfigurationErrors.NONE, libraryCapabilities, SpanTagsPropagator.NOOP_PROPAGATOR ) diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestSuiteImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestSuiteImplTest.groovy index 520ae0e3f2c..c78510db52d 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestSuiteImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/TestSuiteImplTest.groovy @@ -10,6 +10,7 @@ import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.civisibility.codeowners.NoCodeowners +import datadog.trace.civisibility.config.ConfigurationErrors import datadog.trace.civisibility.decorator.TestDecoratorImpl import datadog.trace.civisibility.source.LinesResolver import datadog.trace.civisibility.source.SourcePathResolver @@ -84,7 +85,7 @@ class TestSuiteImplTest extends SpanWriterTest { linesResolver, coverageStoreFactory, executionResults, - false, + ConfigurationErrors.NONE, [], SpanTagsPropagator.NOOP_PROPAGATOR ) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/DDTags.java b/dd-trace-api/src/main/java/datadog/trace/api/DDTags.java index 3c278803d8b..3aecbf1de45 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/DDTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/DDTags.java @@ -77,7 +77,16 @@ public class DDTags { public static final String LIBRARY_VERSION_TAG_KEY = "library_version"; public static final String CI_ENV_VARS = "_dd.ci.env_vars"; public static final String CI_ITR_TESTS_SKIPPED = "_dd.ci.itr.tests_skipped"; - public static final String CI_LIBRARY_CONFIGURATION_ERROR = "_dd.ci.library_configuration_error"; + public static final String CI_LIBRARY_CONFIGURATION_ERROR_SETTINGS = + "_dd.ci.library_configuration_error.settings"; + public static final String CI_LIBRARY_CONFIGURATION_ERROR_SKIPPABLE_TESTS = + "_dd.ci.library_configuration_error.skippable_tests"; + public static final String CI_LIBRARY_CONFIGURATION_ERROR_FLAKY_TESTS = + "_dd.ci.library_configuration_error.flaky_tests"; + public static final String CI_LIBRARY_CONFIGURATION_ERROR_KNOWN_TESTS = + "_dd.ci.library_configuration_error.known_tests"; + public static final String CI_LIBRARY_CONFIGURATION_ERROR_TEST_MANAGEMENT_TESTS = + "_dd.ci.library_configuration_error.test_management_tests"; public static final String TEST_IS_USER_PROVIDED_SERVICE = "_dd.test.is_user_provided_service"; public static final String TEST_HAS_FAILED_TEST_REPLAY = "_dd.test.has_failed_test_replay"; public static final String MEASURED = "_dd.measured";