From 38eb26a18024a128ca9a14e81c1874385e116643 Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Mon, 26 May 2025 19:45:23 -0700 Subject: [PATCH] Improved testing of package test text output --- recipe/build-settings.cake | 2 +- recipe/output-checks.cake | 11 ++++++++--- recipe/package-checks.cake | 18 +----------------- recipe/package-definition.cake | 2 +- recipe/package-test.cake | 2 +- recipe/test-reports.cake | 28 ++++++++++++++++++---------- recipe/test-runners.cake | 8 ++++---- 7 files changed, 34 insertions(+), 37 deletions(-) diff --git a/recipe/build-settings.cake b/recipe/build-settings.cake index ffa475f..4d5cc1c 100644 --- a/recipe/build-settings.cake +++ b/recipe/build-settings.cake @@ -230,7 +230,7 @@ public static class BuildSettings FileName = "build-results/NUnitConsole.binlog", Imports = MSBuildBinaryLoggerImports.Embed } - }.WithProperty("Version", BuildSettings.PackageVersion) + }.WithProperty("Version", PackageVersion) }; // File Header Checks diff --git a/recipe/output-checks.cake b/recipe/output-checks.cake index 1b019a0..bd427e3 100644 --- a/recipe/output-checks.cake +++ b/recipe/output-checks.cake @@ -1,5 +1,5 @@ ////////////////////////////////////////////////////////////////////// -// SYNTAX FOR EXPRESSING A STRING CONSTRAINT +// STATIC SYNTAX FOR EXPRESSING OUTPUT CHECKS ////////////////////////////////////////////////////////////////////// //public static class StringConstraints @@ -22,6 +22,11 @@ // } //} +public static OutputContainsCheck Contains(string expectedText, int atleast = 1, int exactly = -1) + => new OutputContainsCheck(expectedText, atleast, exactly); + +public static OutputDoesNotContain DoesNotContain(string text) => new OutputDoesNotContain(text); + // OutputCheck is used to check content of redirected package test output public abstract class OutputCheck { @@ -43,9 +48,9 @@ public abstract class OutputCheck public string Message { get; protected set; } } -public class OutputContains : OutputCheck +public class OutputContainsCheck : OutputCheck { - public OutputContains(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { } + public OutputContainsCheck(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { } public override bool Matches(string output) { diff --git a/recipe/package-checks.cake b/recipe/package-checks.cake index dcc3858..4d52582 100644 --- a/recipe/package-checks.cake +++ b/recipe/package-checks.cake @@ -1,23 +1,7 @@ ////////////////////////////////////////////////////////////////////// -// SYNTAX FOR EXPRESSING CHECKS +// STATIC SYNTAX FOR EXPRESSING PACKAGE CHECKS ////////////////////////////////////////////////////////////////////// -public static class Check -{ - public static void That(DirectoryPath testDirPath, IList checks) - { - if (checks == null) - throw new ArgumentNullException(nameof(checks)); - - bool allOK = true; - - foreach (var check in checks) - allOK &= check.ApplyTo(testDirPath); - - if (!allOK) throw new Exception("Verification failed!"); - } -} - public static FileCheck HasFile(FilePath file) => HasFiles(new[] { file }); public static FileCheck HasFiles(params FilePath[] files) => new FileCheck(files); diff --git a/recipe/package-definition.cake b/recipe/package-definition.cake index 42be844..0140618 100644 --- a/recipe/package-definition.cake +++ b/recipe/package-definition.cake @@ -242,7 +242,7 @@ public abstract class PackageDefinition string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}"; if (CommandLineOptions.TraceLevel.Value != "Off") arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}"; - bool redirectOutput = packageTest.OutputCheck != null; + bool redirectOutput = packageTest.ExpectedOutput != null; int rc = runner.RunPackageTest(arguments, redirectOutput); diff --git a/recipe/package-test.cake b/recipe/package-test.cake index cfa1d4e..94e2cbc 100644 --- a/recipe/package-test.cake +++ b/recipe/package-test.cake @@ -14,7 +14,7 @@ public class PackageTest public string Arguments { get; set; } public int ExpectedReturnCode { get; set; } = 0; public ExpectedResult ExpectedResult { get; set; } - public OutputCheck OutputCheck { get; set; } + public OutputCheck[] ExpectedOutput { get; set; } public ExtensionSpecifier[] ExtensionsNeeded { get; set; } = new ExtensionSpecifier[0]; public IPackageTestRunner[] TestRunners { get; set; } = new IPackageTestRunner[0]; diff --git a/recipe/test-reports.cake b/recipe/test-reports.cake index 8ec9306..d610171 100644 --- a/recipe/test-reports.cake +++ b/recipe/test-reports.cake @@ -13,7 +13,7 @@ public class PackageTestReport Runner = runner; var expectedResult = test.ExpectedResult; - var expectedOutput = test.OutputCheck; + var expectedOutput = test.ExpectedOutput; if (expectedResult != null) { @@ -57,10 +57,17 @@ public class PackageTestReport } if (expectedOutput != null) - { - if (!expectedOutput.Matches(runner.Output)) - Errors.Add(expectedOutput.Message); - } + { + var output = runner.Output; + if (output is not null) + foreach (var outputCheck in expectedOutput) + { + if (!outputCheck.Matches(output)) + Errors.Add(outputCheck.Message); + } + else + Errors.Add("No output was produced"); + } } public PackageTestReport(PackageTest test, int rc, IPackageTestRunner runner = null) @@ -71,11 +78,12 @@ public class PackageTestReport if (rc != test.ExpectedReturnCode) Errors.Add($" Expected: rc = {test.ExpectedReturnCode} But was: {rc}"); - else if (test.OutputCheck != null) - { - if (!test.OutputCheck.Matches(runner.Output)) - Errors.Add(test.OutputCheck.Message); - } + else if (test.ExpectedOutput != null) + foreach (var outputCheck in test.ExpectedOutput) + { + if (!outputCheck.Matches(runner.Output)) + Errors.Add(outputCheck.Message); + } } public PackageTestReport(PackageTest test, Exception ex, IPackageTestRunner runner = null) diff --git a/recipe/test-runners.cake b/recipe/test-runners.cake index c5ac75f..3141282 100644 --- a/recipe/test-runners.cake +++ b/recipe/test-runners.cake @@ -21,7 +21,7 @@ public interface IPackageTestRunner string PackageId { get; } string Version { get; } - IEnumerable Output { get; } + string Output { get; } int RunPackageTest(string arguments, bool redirectOutput = false); } @@ -44,7 +44,7 @@ public abstract class TestRunner public string PackageId { get; protected set; } public string Version { get; protected set; } - public IEnumerable Output { get; protected set; } + public string Output { get; private set; } protected int RunPackageTest(FilePath executablePath, string arguments = null, bool redirectOutput = false) { @@ -90,14 +90,14 @@ public abstract class TestRunner IEnumerable output; // If Redirected Output was not requested, output will be null int rc = Context.StartProcess(executablePath, processSettings, out output); - Output = output; + Output = output != null ? string.Join("\r\n", output) : null; return rc; } internal string OutputHandler(string output) { // Ensure that package test output displays and is also re-directed. - // If the derive class doesn't need the output, it doesn't retrieve it. + // If the derived class doesn't need the output, it doesn't retrieve it. Console.WriteLine(output); return output; }