diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs index 4ff5bf012a..d94087239c 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs @@ -1,4 +1,5 @@ using Perfolizer.Mathematics.SignificanceTesting; +using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Analysers { @@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh /// Checks distribution against Zero Measurement hypothesis in case of two samples /// /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead) + public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null) { if (workload.Length < 3 || overhead.Length < 3) return false; - return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected; + return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected; } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 0e59568a69..6b71e1e004 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -9,9 +9,12 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; +using Perfolizer.Mathematics.Thresholds; using Xunit; using Xunit.Abstractions; @@ -22,6 +25,8 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor // NativeAot takes a long time to build, so not including it in these tests. // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. + private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); + public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } private static IEnumerable EmptyBenchmarkTypes() => @@ -102,19 +107,22 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkTy private void AssertZeroResults(Type benchmarkType, IConfig config) { var summary = CanExecute(benchmarkType, config - .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); Assert.True(isZero, $"Actual time was not 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); @@ -155,19 +163,22 @@ public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Fr private void AssertDifferentSizedStructsResults(IConfig config) { var summary = CanExecute(config - .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); Assert.False(isZero, $"Actual time was 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");