From 360435f41d5ba1d8e69910bf63b82783d6016c6f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Levesque Date: Fri, 22 Dec 2023 12:46:02 -0500 Subject: [PATCH] benchmarks: Improve benchmark for better precision and run time. --- .github/.commitsar.yml | 2 + .github/PULL_REQUEST_TEMPLATE.md | 1 + .github/workflows/conventional-commits.yml | 2 + build/gitversion.yml | 2 +- src/DynamicMvvm.Benchmarks/Benchmark.cs | 111 --------- .../DynamicMvvm.Benchmarks.csproj | 1 + .../IViewModel.Extensions.Benchmark.cs | 234 ++++++++++++++++++ src/DynamicMvvm.Benchmarks/Program.cs | 22 +- .../TestViewModelBase.cs | 79 ++++++ src/DynamicMvvm.Benchmarks/ViewModel.cs | 14 +- .../ViewModelBase.Benchmark.cs | 57 +++++ 11 files changed, 399 insertions(+), 126 deletions(-) create mode 100644 .github/.commitsar.yml delete mode 100644 src/DynamicMvvm.Benchmarks/Benchmark.cs create mode 100644 src/DynamicMvvm.Benchmarks/IViewModel.Extensions.Benchmark.cs create mode 100644 src/DynamicMvvm.Benchmarks/TestViewModelBase.cs create mode 100644 src/DynamicMvvm.Benchmarks/ViewModelBase.Benchmark.cs diff --git a/.github/.commitsar.yml b/.github/.commitsar.yml new file mode 100644 index 0000000..c29e6b0 --- /dev/null +++ b/.github/.commitsar.yml @@ -0,0 +1,2 @@ +commits: + strict: false \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 791f15f..1fef7d2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -35,6 +35,7 @@ GitHub Issue: # - [ ] **None** (The library is unchanged.) - Only code under the `build` folder was changed. - Only code under the `.github` folder was changed. + - Only code in the Benchmarks project was changed. ## Checklist diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index 54143ef..fb26c86 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -13,3 +13,5 @@ jobs: uses: actions/checkout@v1 - name: Commitsar check uses: docker://aevea/commitsar + env: + COMMITSAR_CONFIG_PATH : ./.github diff --git a/build/gitversion.yml b/build/gitversion.yml index ddc1512..863dfb1 100644 --- a/build/gitversion.yml +++ b/build/gitversion.yml @@ -10,7 +10,7 @@ commit-message-incrementing: Enabled major-version-bump-message: "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\\([\\w\\s-]*\\))?(!:|:.*\\n\\n((.+\\n)+\\n)?BREAKING CHANGE:\\s.+)" minor-version-bump-message: "^(feat)(\\([\\w\\s-]*\\))?:" patch-version-bump-message: "^(build|chore|docs|fix|perf|refactor|revert|style|test)(\\([\\w\\s-]*\\))?:" -no-bump-message: "^(ci)(\\([\\w\\s-]*\\))?:" # You can use the "ci" type to avoid bumping the version when your changes are limited to the build or .github folders. +no-bump-message: "^(ci|benchmarks)(\\([\\w\\s-]*\\))?:" # You can use the "ci" or "benchmarks" type to avoid bumping the version when your changes are limited to the [build or .github folders] or limited to benchmark code. branches: main: regex: ^master$|^main$ diff --git a/src/DynamicMvvm.Benchmarks/Benchmark.cs b/src/DynamicMvvm.Benchmarks/Benchmark.cs deleted file mode 100644 index 70599f8..0000000 --- a/src/DynamicMvvm.Benchmarks/Benchmark.cs +++ /dev/null @@ -1,111 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; -using Chinook.DynamicMvvm; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -namespace DynamicMvvm.Benchmarks; - -[MemoryDiagnoser] -public class Benchmark -{ - private readonly IServiceProvider _serviceProvider = new HostBuilder() - .ConfigureServices(serviceCollection => serviceCollection - .AddSingleton() - .AddSingleton() - ) - .Build() - .Services; - - private ViewModel? _viewModel1; - private ViewModel? _viewModel2; - - [Benchmark] - public void CreateViewModel() - { - var vm = new ViewModel(_serviceProvider); - } - - [Benchmark] - public void CreateViewModel_WithExplicitName() - { - var vm = new ViewModel("ViewModel", _serviceProvider); - } - - [IterationSetup(Targets = new[] - { - nameof(ReadProperty_Unresolved), - nameof(ReadProperty_Resolved), - nameof(SetProperty_Unresolved), - nameof(SetProperty_Resolved), - nameof(DisposeViewModel), - })] - public void SetupViewModel() - { - _viewModel1 = new ViewModel("ViewModel", _serviceProvider); - } - - [IterationSetup(Targets = new[] - { - nameof(SetProperty_Unresolved_WithListener), - nameof(SetProperty_Resolved_WithListener), - nameof(DisposeViewModel_WithListener), - })] - public void SetupViewModelWithListener() - { - _viewModel2 = new ViewModel("ViewModel", _serviceProvider); - _viewModel2.PropertyChanged += (s, e) => { }; - } - - [Benchmark] - public void ReadProperty_Unresolved() - { - var value = _viewModel1!.Number; - } - - [Benchmark] - public void ReadProperty_Resolved() - { - var value = _viewModel1!.NumberResolved; - } - - [Benchmark] - public void SetProperty_Unresolved() - { - _viewModel1!.Number = 1; - } - - [Benchmark] - public void SetProperty_Resolved() - { - _viewModel1!.NumberResolved = 1; - } - - [Benchmark] - public void DisposeViewModel() - { - _viewModel1!.Dispose(); - } - - [Benchmark] - public void SetProperty_Unresolved_WithListener() - { - _viewModel2!.Number = 1; - } - - [Benchmark] - public void SetProperty_Resolved_WithListener() - { - _viewModel2!.NumberResolved = 1; - } - - [Benchmark] - public void DisposeViewModel_WithListener() - { - _viewModel2!.Dispose(); - } -} diff --git a/src/DynamicMvvm.Benchmarks/DynamicMvvm.Benchmarks.csproj b/src/DynamicMvvm.Benchmarks/DynamicMvvm.Benchmarks.csproj index 7f2856d..51e3547 100644 --- a/src/DynamicMvvm.Benchmarks/DynamicMvvm.Benchmarks.csproj +++ b/src/DynamicMvvm.Benchmarks/DynamicMvvm.Benchmarks.csproj @@ -11,6 +11,7 @@ + diff --git a/src/DynamicMvvm.Benchmarks/IViewModel.Extensions.Benchmark.cs b/src/DynamicMvvm.Benchmarks/IViewModel.Extensions.Benchmark.cs new file mode 100644 index 0000000..c0a0f28 --- /dev/null +++ b/src/DynamicMvvm.Benchmarks/IViewModel.Extensions.Benchmark.cs @@ -0,0 +1,234 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reactive.Linq; +using System.Reflection.Emit; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using Chinook.DynamicMvvm; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace DynamicMvvm.Benchmarks +{ + [MemoryDiagnoser] + [MaxIterationCount(36)] + [MaxWarmupCount(16)] + public class ViewModelExtensionsBenchmark + { + internal static readonly IServiceProvider ServiceProvider = new HostBuilder() + .ConfigureServices(serviceCollection => serviceCollection + .AddSingleton() + .AddSingleton() + ) + .Build() + .Services; + + internal static readonly Type DynamicViewModelType = GetDynamicViewModelType(); + private const int PropertyCount = 32; + private const int ViewModelCount = 1024 * 1024; + private static string[] propertyNames = Enumerable.Range(0, PropertyCount).Select(i => $"Number{i}").ToArray(); + + /// + /// Generates a dynamic type that inherits from and has properties. + /// + /// The amount of property defined in the dynamic class. + /// A dynamically generated type. + internal static Type GetDynamicViewModelType(int propertyCount = PropertyCount) + { + // Create a new dynamic assembly + AssemblyName assemblyName = new AssemblyName("DynamicAssembly"); + AssemblyBuilder assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + + // Create a new module within the assembly + ModuleBuilder moduleBuilder = assemblyBuilder.DefineDynamicModule("DynamicModule"); + + // Create a new type that inherits from ViewModelBase + TypeBuilder typeBuilder = moduleBuilder.DefineType("DynamicClass", TypeAttributes.Public | TypeAttributes.Class, typeof(ViewModelBase)); + + // Define a parameterless constructor + ConstructorBuilder constructorBuilder = typeBuilder.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard, new Type[] { typeof(string), typeof(IServiceProvider) }); + ILGenerator constructorIL = constructorBuilder.GetILGenerator(); + constructorIL.Emit(OpCodes.Ldarg_0); + constructorIL.Emit(OpCodes.Ldarg_1); + constructorIL.Emit(OpCodes.Ldarg_2); + constructorIL.Emit(OpCodes.Call, typeof(ViewModelBase).GetConstructor(new Type[] { typeof(string), typeof(IServiceProvider) })!); + constructorIL.Emit(OpCodes.Ret); + + // There is a name conflict on IViewModelExtensions which requires us to use reflection to get it (because typeof(IViewModelExtensions) is ambiguous). + var viewModelExtensionsType = Assembly.GetAssembly(typeof(IViewModel))!.GetType("Chinook.DynamicMvvm.IViewModelExtensions")!; + + for (int i = 0; i < propertyCount; i++) + { + // Define a 'NumberX' property with the specified getter and setter + PropertyBuilder propertyBuilder = typeBuilder.DefineProperty("Number" + i, PropertyAttributes.None, typeof(int), null); + + MethodBuilder getMethod = typeBuilder.DefineMethod("get_Number" + i, MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig, typeof(int), Type.EmptyTypes); + ILGenerator getMethodIL = getMethod.GetILGenerator(); + getMethodIL.Emit(OpCodes.Ldarg_0); + getMethodIL.Emit(OpCodes.Ldc_I4_S, 42); // Initial value + getMethodIL.Emit(OpCodes.Ldstr, "Number" + i); + var getMethodInfo = viewModelExtensionsType.GetMethods() + .Where(m => m.Name == "Get" && m.IsGenericMethod) + .First(m => + { + var parameters = m.GetParameters(); + return parameters.Length == 3 && parameters[0].ParameterType == typeof(IViewModel) && parameters[2].ParameterType == typeof(string); + }); + getMethodIL.EmitCall(OpCodes.Call, getMethodInfo.MakeGenericMethod(typeof(int)), null); + getMethodIL.Emit(OpCodes.Ret); + + MethodBuilder setMethod = typeBuilder.DefineMethod("set_Number" + i, MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig, null, new Type[] { typeof(int) }); + ILGenerator setMethodIL = setMethod.GetILGenerator(); + setMethodIL.Emit(OpCodes.Ldarg_0); + setMethodIL.Emit(OpCodes.Ldarg_1); + setMethodIL.Emit(OpCodes.Ldstr, "Number" + i); + var setMethodInfo = viewModelExtensionsType.GetMethods() + .Where(m => m.Name == "Set" && m.IsGenericMethod) + .First(m => + { + var parameters = m.GetParameters(); + return parameters.Length == 3 && parameters[0].ParameterType == typeof(IViewModel) && parameters[2].ParameterType == typeof(string); + }); + setMethodIL.EmitCall(OpCodes.Call, setMethodInfo.MakeGenericMethod(typeof(int)), null); + setMethodIL.Emit(OpCodes.Ret); + + propertyBuilder.SetGetMethod(getMethod); + propertyBuilder.SetSetMethod(setMethod); + } + + // Create the type + Type dynamicType = typeBuilder.CreateType(); + + return dynamicType; + } + + /// + /// This vm always initializes new property instances when being invoked. + /// + private NeverInitiatedViewModel _neverInitiatedVM = new(); + + private InitiatedViewModel _initiatedVM = new(); + + private IViewModel[]? _vmsForPropertySetter; + private int _i = 0; + + [GlobalSetup(Target = nameof(Set_Unresolved))] + public void SetupVMForPropertySetter() + { + _i = 0; + _vmsForPropertySetter = new IViewModel[ViewModelCount]; + for (int i = 0; i < ViewModelCount; i++) + { + _vmsForPropertySetter[i] = (IViewModel)Activator.CreateInstance(DynamicViewModelType, "ViewModelName", ServiceProvider)!; + } + } + + [Benchmark] + public int GetFromValue_Unresolved() + { + return _neverInitiatedVM.Number; + } + + [Benchmark] + public int GetFromObservable_Unresolved() + { + return _neverInitiatedVM.ObservableNumber; + } + + [Benchmark] + public int GetFromValue_Resolved() + { + return _initiatedVM.Number; + } + + [Benchmark] + public int GetFromObservable_Resolved() + { + return _initiatedVM.ObservableNumber; + } + + [Benchmark(OperationsPerInvoke = PropertyCount)] + [MaxIterationCount(24)] + public void Set_Unresolved() + { + var i = Interlocked.Increment(ref _i); + var vm = _vmsForPropertySetter![i]; + for (int propertyIndex = 0; propertyIndex < PropertyCount; propertyIndex++) + { + vm!.Set(i, propertyNames[propertyIndex]); + } + } + + [Benchmark] + public void Set_Resolved() + { + _initiatedVM.Number = 1; + } + } + + public sealed class NeverInitiatedViewModel : TestViewModelBase + { + public NeverInitiatedViewModel(IServiceProvider? serviceProvider = null) + : base(serviceProvider ?? ViewModelExtensionsBenchmark.ServiceProvider) + { + } + + public int Number + { + get => this.Get(initialValue: 42); + set => this.Set(value); + } + + public int ObservableNumber + { + get => this.GetFromObservable(Observable.Never(), initialValue: 0); + set => this.Set(value); + } + } + + public sealed class InitiatedViewModel : TestViewModelWithProperty + { + public InitiatedViewModel() + { + ServiceProvider = ViewModelExtensionsBenchmark.ServiceProvider; + + Resolve(Number); + Resolve(ObservableNumber); + } + + public int Number + { + get => this.Get(initialValue: 42); + set => this.Set(value); + } + + public int ObservableNumber + { + get => this.GetFromObservable(Observable.Never(), initialValue: 0); + set => this.Set(value); + } + } + + public class TestViewModelWithProperty : TestViewModelBase + { + IDictionary _disposables = new Dictionary(); + + protected void Resolve(object value) + { + } + + public override void AddDisposable(string key, IDisposable disposable) + { + _disposables[key] = disposable; + } + + public override bool TryGetDisposable(string key, out IDisposable? disposable) + { + return _disposables.TryGetValue(key, out disposable); + } + } +} diff --git a/src/DynamicMvvm.Benchmarks/Program.cs b/src/DynamicMvvm.Benchmarks/Program.cs index fb0f6a8..9f5ec86 100644 --- a/src/DynamicMvvm.Benchmarks/Program.cs +++ b/src/DynamicMvvm.Benchmarks/Program.cs @@ -3,11 +3,25 @@ using Chinook.DynamicMvvm; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Order; -BenchmarkRunner.Run(); +BenchmarkRunner.Run(new[] + { + typeof(ViewModelBaseBenchmark), + typeof(ViewModelExtensionsBenchmark), + }, + ManualConfig + .Create(DefaultConfig.Instance) + .WithOptions(ConfigOptions.JoinSummary) + .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Declared, MethodOrderPolicy.Declared)) + .HideColumns("Type", "Job", "InvocationCount", "UnrollFactor", "Error", "StdDev", "MaxIterationCount", "MaxWarmupIterationCount") +); // The following section is to profile manually using Visual Studio's debugger. +//Console.ReadKey(); + //var serviceProvider = new HostBuilder() // .ConfigureServices(serviceCollection => serviceCollection // .AddSingleton() @@ -16,7 +30,13 @@ // .Build() // .Services; +//var vm = new InitiatedViewModel(); +//vm.Number = 1; + //var vm1 = new ViewModel("ViewModel", serviceProvider); //var vm2 = new ViewModel("ViewModel", serviceProvider); +//var value = vm1.NumberResolved; +//value = vm1.Number; +//Console.WriteLine(value); //Console.Read(); diff --git a/src/DynamicMvvm.Benchmarks/TestViewModelBase.cs b/src/DynamicMvvm.Benchmarks/TestViewModelBase.cs new file mode 100644 index 0000000..bc8d80b --- /dev/null +++ b/src/DynamicMvvm.Benchmarks/TestViewModelBase.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Chinook.DynamicMvvm; +using Microsoft.Extensions.DependencyInjection; + +namespace DynamicMvvm.Benchmarks +{ + /// + /// This implementation of IViewModel is used for testing the extension methods of IViewModel. + /// It's not a valid implementation of IViewModel. + /// + public class TestViewModelBase : IViewModel + { + public TestViewModelBase(IServiceProvider? serviceProvider = null) + { + ServiceProvider = serviceProvider; + } + + public string Name => "TestViewModelBase"; + + public virtual IEnumerable> Disposables => Enumerable.Empty>(); + + public IDispatcher? Dispatcher { get; set; } + + public IServiceProvider? ServiceProvider { get; set; } + + public bool IsDisposed { get; set; } + + public bool HasErrors => false; + + public event Action? DispatcherChanged; + public event PropertyChangedEventHandler? PropertyChanged; + public event EventHandler? ErrorsChanged; + + public virtual void AddDisposable(string key, IDisposable disposable) + { + } + + public void ClearErrors(string? propertyName = null) + { + } + + public void Dispose() + { + } + + public IEnumerable GetErrors(string? propertyName) + { + return Enumerable.Empty(); + } + + public void RaisePropertyChanged(string propertyName) + { + } + + public void RemoveDisposable(string key) + { + } + + public void SetErrors(IDictionary> errors) + { + } + + public void SetErrors(string propertyName, IEnumerable errors) + { + } + + public virtual bool TryGetDisposable(string key, out IDisposable? disposable) + { + disposable = default; + return false; + } + } +} diff --git a/src/DynamicMvvm.Benchmarks/ViewModel.cs b/src/DynamicMvvm.Benchmarks/ViewModel.cs index a83e880..f735c06 100644 --- a/src/DynamicMvvm.Benchmarks/ViewModel.cs +++ b/src/DynamicMvvm.Benchmarks/ViewModel.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reactive.Linq; using System.Text; using System.Threading.Tasks; using Chinook.DynamicMvvm; @@ -12,23 +13,10 @@ public class ViewModel : ViewModelBase public ViewModel(string? name, IServiceProvider serviceProvider) : base(name, serviceProvider) { - var value = NumberResolved; } public ViewModel(IServiceProvider serviceProvider) : this(name: default, serviceProvider: serviceProvider) { } - - public int Number - { - get => this.Get(initialValue: 42); - set => this.Set(value); - } - - public int NumberResolved - { - get => this.Get(initialValue: 42); - set => this.Set(value); - } } diff --git a/src/DynamicMvvm.Benchmarks/ViewModelBase.Benchmark.cs b/src/DynamicMvvm.Benchmarks/ViewModelBase.Benchmark.cs new file mode 100644 index 0000000..6305486 --- /dev/null +++ b/src/DynamicMvvm.Benchmarks/ViewModelBase.Benchmark.cs @@ -0,0 +1,57 @@ +using BenchmarkDotNet.Attributes; +using Chinook.DynamicMvvm; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace DynamicMvvm.Benchmarks; + +[MemoryDiagnoser] +[MaxIterationCount(36)] +[MaxWarmupCount(16)] +public class ViewModelBaseBenchmark +{ + private readonly IServiceProvider _serviceProvider = new HostBuilder() + .ConfigureServices(serviceCollection => serviceCollection + .AddSingleton() + .AddSingleton() + ) + .Build() + .Services; + + private const int ViewModelCount = 2500000; + private ViewModel[]? _viewModelsToDispose; + + [Benchmark] + public IViewModel CreateViewModel() + { + return new ViewModel(_serviceProvider); + } + + [Benchmark] + public IViewModel CreateViewModel_WithExplicitName() + { + return new ViewModel("ViewModel", _serviceProvider); + } + + [IterationSetup(Targets = new[] + { + nameof(DisposeViewModel), + })] + public void SetupViewModel() + { + _viewModelsToDispose = Enumerable + .Range(0, ViewModelCount) + .Select(i => new ViewModel("ViewModel", _serviceProvider)) + .ToArray(); + } + + [Benchmark(OperationsPerInvoke = ViewModelCount)] + [MaxIterationCount(16)] + public void DisposeViewModel() + { + for (var i = 0; i < ViewModelCount; i++) + { + _viewModelsToDispose![i].Dispose(); + } + } +}