diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
index 4ff5bf012a..d94087239c 100644
--- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
+++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
@@ -1,4 +1,5 @@
using Perfolizer.Mathematics.SignificanceTesting;
+using Perfolizer.Mathematics.Thresholds;
namespace BenchmarkDotNet.Analysers
{
@@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh
/// Checks distribution against Zero Measurement hypothesis in case of two samples
///
/// True if measurement is ZeroMeasurement
- public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead)
+ public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null)
{
if (workload.Length < 3 || overhead.Length < 3)
return false;
- return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected;
+ return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
}
}
}
\ No newline at end of file
diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
index 0e59568a69..6b71e1e004 100644
--- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
+++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
@@ -9,9 +9,12 @@
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Portability;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Tests.XUnit;
using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Perfolizer.Horology;
+using Perfolizer.Mathematics.Thresholds;
using Xunit;
using Xunit.Abstractions;
@@ -22,6 +25,8 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
// NativeAot takes a long time to build, so not including it in these tests.
// We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
+ private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
+
public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
private static IEnumerable EmptyBenchmarkTypes() =>
@@ -102,19 +107,22 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkTy
private void AssertZeroResults(Type benchmarkType, IConfig config)
{
var summary = CanExecute(benchmarkType, config
- .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond))
+ .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
);
+ var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
+ var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
+
foreach (var report in summary.Reports)
{
var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
- bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements);
+ bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
Assert.True(isZero, $"Actual time was not 0.");
- isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements);
+ isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
Assert.True(isZero, "Overhead took more time than workload.");
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
@@ -155,19 +163,22 @@ public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Fr
private void AssertDifferentSizedStructsResults(IConfig config)
{
var summary = CanExecute(config
- .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond))
+ .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
);
+ var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
+ var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
+
foreach (var report in summary.Reports)
{
var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
- bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements);
+ bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
Assert.False(isZero, $"Actual time was 0.");
- isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements);
+ isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
Assert.True(isZero, "Overhead took more time than workload.");
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");