Customizing Runtime

Currently, we have only information about customizing Mono in this section. If you want to customize .NET Core, read an article about Toolchains.

Sample: IntroCustomMono

BenchmarkDotNet allows you to compare different runtimes, including Mono. If you apply [MonoJob] attribute to your class we use your default mono runtime. If you want to compare different versions of Mono you need to provide use the custom paths. You can do this today by using the overloaded ctor of MonoJob attribute or by specifying the runtime in a fluent way.

The mono runtime can also operate as an ahead-of-time compiler. Using mono's AOT mode requires providing the AOT compilation arguments, as well as the path to mono's corlib. (See IntroCustomMonoObjectStyleAot in the below example).

Source code

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Running;

namespace BenchmarkDotNet.Samples
{
    // *** Attribute Style ***

    [MonoJob("Mono x64", @"C:\Program Files\Mono\bin\mono.exe")]
    [MonoJob("Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")]
    public class IntroCustomMono
    {
        [Benchmark]
        public void Foo()
        {
            // Benchmark body
        }
    }

    // *** Object Style ***

    [Config(typeof(Config))]
    public class IntroCustomMonoObjectStyle
    {
        private class Config : ManualConfig
        {
            public Config()
            {
                AddJob(Job.ShortRun.WithRuntime(new MonoRuntime(
                    "Mono x64", @"C:\Program Files\Mono\bin\mono.exe")));
                AddJob(Job.ShortRun.WithRuntime(new MonoRuntime(
                    "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")));
            }
        }

        [Benchmark]
        public void Foo()
        {
            // Benchmark body
        }
    }

    // ** Object Style, Using AOT **

    [Config(typeof(Config))]
    public class IntroCustomMonoObjectStyleAot
    {
        private class Config : ManualConfig
        {
            public void AddMono (string name, string mono_top_dir)
            {
                var aot_compile_args  = "--aot=llvm";
                var mono_bcl = $@"{mono_top_dir}\lib\mono\4.5";
                var mono_bin = $@"{mono_top_dir}\bin\mono.exe";
                AddJob(Job.ShortRun.WithRuntime(new MonoRuntime(
                    name, mono_bin, aot_compile_args, mono_bcl)));
            }

            public Config()
            {
                AddMono("Mono x64", @"C:\Program Files\Mono");
                AddMono("Mono x86", @"C:\Program Files (x86)\Mono");
            }
        }

        [Benchmark]
        public void Foo()
        {
            // Benchmark body
        }
    }

    // *** Fluent Config ***

    public class IntroCustomMonoFluentConfig
    {
        public static void Run()
        {
            BenchmarkRunner.Run<IntroCustomMonoFluentConfig>(ManualConfig
                .CreateMinimumViable()
                .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime(
                    "Mono x64", @"C:\Program Files\Mono\bin\mono.exe")))
                .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime(
                    "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe"))));
        }

        [Benchmark]
        public void Foo()
        {
            // Benchmark body
        }
    }
}

Sample: IntroCustomMonoArguments

Source code

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;

namespace BenchmarkDotNet.Samples
{
    [Config(typeof(ConfigWithCustomArguments))]
    public class IntroCustomMonoArguments
    {
        public class ConfigWithCustomArguments : ManualConfig
        {
            public ConfigWithCustomArguments()
            {
                // --optimize=MODE , -O=mode
                // MODE is a comma separated list of optimizations. They also allow
                // optimizations to be turned off by prefixing the optimization
                // name with a minus sign.

                AddJob(Job.Default
                    .WithRuntime(MonoRuntime.Default)
                    .WithArguments(new[] { new MonoArgument("--optimize=inline") })
                    .WithId("Inlining enabled"));
                AddJob(Job.Default
                    .WithRuntime(MonoRuntime.Default)
                    .WithArguments(new[] { new MonoArgument("--optimize=-inline") })
                    .WithId("Inlining disabled"));
            }
        }

        [Benchmark]
        public void Sample()
        {
            ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined();
            ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined();
            ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined();
        }

        private void ShouldGetInlined() { }
    }
}

Output

| Method |               Job |          Arguments |       Mean |    StdDev |
|------- |------------------ |------------------- |-----------:|----------:|
| Sample | Inlining disabled | --optimize=-inline | 19.4252 ns | 0.4525 ns |
| Sample |  Inlining enabled |  --optimize=inline |  0.0000 ns | 0.0000 ns |

Sample: IntroEnvVars

You can configure custom environment variables for the process that is running your benchmarks. One reason for doing this might be checking out how different runtime knobs affect the performance of .NET Core.

Source code

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;

namespace BenchmarkDotNet.Samples
{
    [Config(typeof(ConfigWithCustomEnvVars))]
    public class IntroEnvVars
    {
        private class ConfigWithCustomEnvVars : ManualConfig
        {
            private const string JitNoInline = "COMPlus_JitNoInline";

            public ConfigWithCustomEnvVars()
            {
                AddJob(Job.Default.WithRuntime(CoreRuntime.Core21).WithId("Inlining enabled"));
                AddJob(Job.Default.WithRuntime(CoreRuntime.Core21)
                    .WithEnvironmentVariables(new EnvironmentVariable(JitNoInline, "1"))
                    .WithId("Inlining disabled"));
            }
        }

        [Benchmark]
        public void Foo()
        {
            // Benchmark body
        }
    }
}

Sample: IntroStaThread

If the code you want to benchmark requires [System.STAThread] then you need to apply this attribute to the benchmarked method. BenchmarkDotNet will generate executable with [STAThread] applied to it's Main method.

Currently it does not work for .NET Core 2.0 due to this bug.

Source code

using System.Threading;
using BenchmarkDotNet.Attributes;

namespace BenchmarkDotNet.Samples
{
    public class IntroStaThread
    {
        [Benchmark, System.STAThread]
        public void CheckForSTA()
        {
            if (Thread.CurrentThread.GetApartmentState() != ApartmentState.STA)
            {
                throw new ThreadStateException(
                    "The current threads apartment state is not STA");
            }
        }
    }
}