diff --git a/CHANGELOG.md b/CHANGELOG.md index 43a91cd..c7a420f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 2.2.2 +- added generated source code output in verbose mode + ## 2.2.1 - tested with OTP 26 and 27 - updated to argparse 2.0.0 diff --git a/CLI.md b/CLI.md index 70c99c1..299bb10 100644 --- a/CLI.md +++ b/CLI.md @@ -9,22 +9,22 @@ erlperf [FLAG] runner [INIT] [INIT_RUNNER] [DONE] [runner...] ## Flags -| Short | Long | Description | -|-------|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -c | --concurrency | Specifies the number of workers per job. Allowed only in continuous mode | -| | --cv | Coefficient of variation. Accepted in continuous and concurrency estimation mode. Benchmark keeps running until standard deviation is below the specified number | -| -i | --isolation | Requests to run every benchmark in a separate Erlang VM for isolation purposes | -| -s | --samples | Number of samples to take. Defaults to 1 for timed mode, 3 for continuous and concurrency estimation | -| -d | --sample_duration | Sample duration, in milliseconds, for continuous and concurrency estimation modes | -| -l | --loop | Sample duration (iterations) for the timed mode. Engages timed mode when specified | -| | --max | Maximum number of workers allowed in the concurrency estimation mode | -| | --min | Starting number of workers in concurrency estimation mode | -| -pa | | Adds extra code path to the Erlang VM. Useful for benchmarking *.beam files on your filesystem | -| -r | --report | Requests `basic`, `extended` or `full` report. Defaults to `basic` when less than 10 samples are requested, and `extended` for 10 and more | -| -q | -squeeze | Engages concurrency estimation mode | -| -t | -threshold | Sets number of extra workers to try in concurrency estimation mode before concluding the test | -| -v | --verbose | Turns on verbose logging (VM statistics and performance of continuous jobs) | -| -w | --warmup | Warmup | +| Short | Long | Description | +|-------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -c | --concurrency | Specifies the number of workers per job. Allowed only in continuous mode | +| | --cv | Coefficient of variation. Accepted in continuous and concurrency estimation mode. Benchmark keeps running until standard deviation is below the specified number | +| -i | --isolation | Requests to run every benchmark in a separate Erlang VM for isolation purposes | +| -s | --samples | Number of samples to take. Defaults to 1 for timed mode, 3 for continuous and concurrency estimation | +| -d | --sample_duration | Sample duration, in milliseconds, for continuous and concurrency estimation modes | +| -l | --loop | Sample duration (iterations) for the timed mode. Engages timed mode when specified | +| | --max | Maximum number of workers allowed in the concurrency estimation mode | +| | --min | Starting number of workers in concurrency estimation mode | +| -pa | | Adds extra code path to the Erlang VM. Useful for benchmarking *.beam files on your filesystem | +| -r | --report | Requests `basic`, `extended` or `full` report. Defaults to `basic` when less than 10 samples are requested, and `extended` for 10 and more | +| -q | -squeeze | Engages concurrency estimation mode | +| -t | -threshold | Sets number of extra workers to try in concurrency estimation mode before concluding the test | +| -v | --verbose | Turns on verbose mode, including generated source code, VM statistics and performance of continuous jobs) | +| -w | --warmup | Warmup | ## Benchmark code At least one runner code is required. Specify multiple runner codes to perform diff --git a/src/erlperf.app.src b/src/erlperf.app.src index 06bcfc4..f06f921 100644 --- a/src/erlperf.app.src +++ b/src/erlperf.app.src @@ -1,6 +1,6 @@ {application, erlperf, [{description, "Erlang Performance & Benchmarking Suite"}, - {vsn, "2.2.1"}, + {vsn, "2.2.2"}, {registered, [ erlperf_sup, erlperf_job_sup, erlperf_monitor, erlperf_history, erlperf_file_log, erlperf_cluster_monitor diff --git a/src/erlperf_cli.erl b/src/erlperf_cli.erl index 9006635..804e804 100644 --- a/src/erlperf_cli.erl +++ b/src/erlperf_cli.erl @@ -204,6 +204,11 @@ determine_mode(ParsedOpts) -> benchmark(Codes, RunOpts, ConcurrencyTestOpts, false) -> erlperf:benchmark(Codes, RunOpts, ConcurrencyTestOpts); benchmark(Codes, RunOpts, ConcurrencyTestOpts, true) -> + [begin + io:format(">>>>>>>>>>>>>>> ~-32ts ~n", [format_code(maps:get(runner, C))]), + [io:format("~ts~n", [L]) || L <- erlperf_job:source(C)], + io:format("<<<<<<<<<<<<<<< ~n") + end|| C <- Codes], {ok, Pg} = pg:start_link(erlperf), {ok, Monitor} = erlperf_monitor:start_link(), {ok, Logger} = erlperf_file_log:start_link(), diff --git a/src/erlperf_job.erl b/src/erlperf_job.erl index cf2b208..610c6db 100644 --- a/src/erlperf_job.erl +++ b/src/erlperf_job.erl @@ -297,12 +297,30 @@ sample({Module, Arity}) -> {call_count, Count} = erlang:trace_info({Module, Module, Arity}, call_count), Count. +%%% Internal, not exported, record. +-record(exec, { + name :: atom(), %% generated module name (must be generated for tracing to work) + source :: [string()], %% module source code + binary :: binary(), %% generated bytecode + init :: fun(() -> term()), %% init function + init_runner :: fun((term()) -> term()), %% must accept 1 argument + runner :: {fun((term()) -> term()), non_neg_integer()}, + sample_runner :: {fun((non_neg_integer(), term()) -> term()), non_neg_integer()}, + done :: fun((term()) -> term()) %% must accept 1 argument +}). + +-type exec() :: #exec{}. + %% @doc -%% Returns the source code that was generated for this job. --spec source(server_ref()) -> [string()]. +%% Returns the source code generated from the code map, or for a running job. +-spec source(server_ref() | code_map()) -> [string()]. +source(Code) when is_map(Code) -> + #exec{source = Src} = generate(benchmark, Code), + Src; source(JobId) -> gen_server:call(JobId, source). + %% @doc %% Sets job process priority when there are workers running. %% @@ -324,19 +342,6 @@ set_priority(JobId, Priority) -> -include_lib("kernel/include/logger.hrl"). --record(exec, { - name :: atom(), %% generated module name (must be generated for tracing to work) - source :: [string()], %% module source code - binary :: binary(), %% generated bytecode - init :: fun(() -> term()), %% init function - init_runner :: fun((term()) -> term()), %% must accept 1 argument - runner :: {fun((term()) -> term()), non_neg_integer()}, - sample_runner :: {fun((non_neg_integer(), term()) -> term()), non_neg_integer()}, - done :: fun((term()) -> term()) %% must accept 1 argument -}). - --type exec() :: #exec{}. - -record(erlperf_job_state, { %% original spec exec :: exec(), @@ -515,12 +520,15 @@ generate(Name, #{runner := Runner} = Code) -> {IRFun, IRArity, IRExport, IRText} = generate_one(Name, init_runner, maps:get(init_runner, Code, error)), {DoneFun, DoneArity, DoneExport, DoneText} = generate_one(Name, done, maps:get(done, Code, error)), + %% Separator: CR/LF + Sep = io_lib:format("~n", []), + %% RunnerArity: how many arguments _original_ runner wants to accept. %% Example: run(State) is 1, and run() is 0. %% Pass two function names: one that is for sample_count, and one for continuous ContName = atom_to_list(Name), - SampleCountName = list_to_atom(ContName ++ "_" ++ integer_to_list(erlang:unique_integer([positive]))), - {RunnerFun, SampleRunnerFun, RunnerArity, RunArity, RunnerText} = generate_runner(Name, SampleCountName, Runner), + SampleCountName = list_to_atom(ContName ++ "_finite"), + {RunnerFun, SampleRunnerFun, RunnerArity, RunArity, RunnerText} = generate_runner(Name, SampleCountName, Runner, Sep), RunnerExports = [{Name, RunArity}, {SampleCountName, RunArity + 1}], %% verify compatibility between 4 pieces of code @@ -538,9 +546,9 @@ generate(Name, #{runner := Runner} = Code) -> Exports = lists:concat(lists:join(", ", [io_lib:format("~s/~b", [F, Arity]) || {F, Arity} <- [InitExport, IRExport, DoneExport | RunnerExports], Arity =/= undefined])), - Texts = [Text || Text <- [InitText, IRText, DoneText | RunnerText], Text =/= ""], + Texts = [Text ++ Sep || Text <- [InitText, IRText, DoneText | RunnerText], Text =/= ""], - Source = ["-module(" ++ atom_to_list(Name) ++ ").", "-export([" ++ Exports ++ "])." | Texts], + Source = ["-module(" ++ atom_to_list(Name) ++ ")." ++ Sep, "-export([" ++ Exports ++ "])." ++ Sep | Texts], #exec{name = Name, binary = compile(Name, Source), init = InitFun, init_runner = IRFun, source = Source, runner = {RunnerFun, RunArity}, sample_runner = {SampleRunnerFun, RunArity}, done = DoneFun}. @@ -589,47 +597,47 @@ generate_one(Mod, FunName, Text) when is_list(Text) -> %% runner wrapper: %% Generates at least 2 functions, one for continuous, and one for %% sample-count benchmarking. -generate_runner(Mod, SampleCountName, Fun) when is_function(Fun, 0) -> +generate_runner(Mod, SampleCountName, Fun, Sep) when is_function(Fun, 0) -> { fun (_Ignore) -> Mod:Mod(Fun) end, fun (SampleCount, _Ignore) -> Mod:SampleCountName(SampleCount, Fun) end, 0, 1, - [lists:concat([Mod, "(Fun) -> Fun(), ", Mod, "(Fun)."]), - lists:concat([SampleCountName, "(0, _Fun) -> ok; ", SampleCountName, "(Count, Fun) -> Fun(), ", - SampleCountName, "(Count - 1, Fun)."])] + [lists:concat([Mod, "(Fun) -> Fun(),", Sep, " ", Mod, "(Fun)."]), + lists:concat([SampleCountName, "(0, _Fun) ->", Sep, " ok;", Sep, SampleCountName, "(Count, Fun) ->", Sep, " Fun(),", + Sep, " ", SampleCountName, "(Count - 1, Fun)."])] }; -generate_runner(Mod, SampleCountName, Fun) when is_function(Fun, 1) -> +generate_runner(Mod, SampleCountName, Fun, Sep) when is_function(Fun, 1) -> { fun (Init) -> Mod:Mod(Init, Fun) end, fun (SampleCount, Init) -> Mod:SampleCountName(SampleCount, Init, Fun) end, 1, 2, - [lists:concat([Mod, "(Init, Fun) -> Fun(Init), ", Mod, "(Init, Fun)."]), - lists:concat([SampleCountName, "(0, _Init, _Fun) -> ok; ", SampleCountName, "(Count, Init, Fun) -> Fun(Init), ", - SampleCountName, "(Count - 1, Init, Fun)."])] + [lists:concat([Mod, "(Init, Fun) ->", Sep, " Fun(Init),", Sep, " ", Mod, "(Init, Fun)."]), + lists:concat([SampleCountName, "(0, _Init, _Fun) ->", Sep, " ok;", Sep, SampleCountName, + "(Count, Init, Fun) ->", Sep, " Fun(Init),", Sep, " ", SampleCountName, "(Count - 1, Init, Fun)."])] }; -generate_runner(Mod, SampleCountName, Fun) when is_function(Fun, 2) -> +generate_runner(Mod, SampleCountName, Fun, Sep) when is_function(Fun, 2) -> { fun (Init) -> Mod:Mod(Init, Init, Fun) end, fun (SampleCount, Init) -> Mod:SampleCountName(SampleCount, Init, Init, Fun) end, 2, 3, - [lists:concat([Mod, "(Init, State, Fun) -> ", Mod, "(Init, Fun(Init, State), Fun)."]), - lists:concat([SampleCountName, "(0, _Init, _State, _Fun) -> ok; ", SampleCountName, "(Count, Init, State, Fun) -> ", - SampleCountName, "(Count - 1, Init, Fun(Init, State), Fun)."])] + [lists:concat([Mod, "(Init, State, Fun) ->", Sep, " ", Mod, "(Init, Fun(Init, State), Fun)."]), + lists:concat([SampleCountName, "(0, _Init, _State, _Fun) ->", Sep, " ok; ", SampleCountName, "(Count, Init, State, Fun) ->", + Sep, " ", SampleCountName, "(Count - 1, Init, Fun(Init, State), Fun)."])] }; %% runner wrapper: MFA -generate_runner(Mod, SampleCountName, {M, F, Args}) when is_atom(M), is_atom(F), is_list(Args) -> +generate_runner(Mod, SampleCountName, {M, F, Args}, Sep) when is_atom(M), is_atom(F), is_list(Args) -> { fun (_Ignore) -> Mod:Mod(M, F, Args) end, fun (SampleCount, _Ignore) -> Mod:SampleCountName(SampleCount, M, F, Args) end, 0, 3, - [lists:concat([Mod, "(M, F, A) -> erlang:apply(M, F, A), ", Mod, "(M, F, A)."]), - lists:concat([SampleCountName, "(0, _M, _F, _A) -> ok; ", SampleCountName, - "(Count, M, F, A) -> erlang:apply(M, F, A), ", SampleCountName, "(Count - 1, M, F, A)."])] + [lists:concat([Mod, "(M, F, A) ->", Sep, " erlang:apply(M, F, A), ", Mod, "(M, F, A)."]), + lists:concat([SampleCountName, "(0, _M, _F, _A) ->", Sep, " ok;", Sep, SampleCountName, + "(Count, M, F, A) ->", Sep, " erlang:apply(M, F, A), ", Sep, " ", SampleCountName, "(Count - 1, M, F, A)."])] }; %% runner wrapper: MFAList -generate_runner(Mod, SampleCountName, [{M, F, Args} | _Tail] = MFAList) when is_atom(M), is_atom(F), is_list(Args) -> +generate_runner(Mod, SampleCountName, [{M, F, Args} | _Tail] = MFAList, Sep) when is_atom(M), is_atom(F), is_list(Args) -> [erlang:error({generate, {runner, 0, invalid, {M1, F1, A}}}) || {M1, F1, A} <- MFAList, not is_atom(M1) orelse not is_atom(F1) orelse not is_list(A)], { @@ -637,11 +645,12 @@ generate_runner(Mod, SampleCountName, [{M, F, Args} | _Tail] = MFAList) when is_ fun (SampleCount, _Ignore) -> Mod:SampleCountName(SampleCount, MFAList) end, 0, 1, [lists:concat([Mod, "(MFAList) -> [erlang:apply(M, F, A) || {M, F, A} <- MFAList], ", Mod, "(MFAList)."]), - lists:concat([SampleCountName, "(0, _MFAList) -> ok; ", SampleCountName, - "(Count, MFAList) -> [erlang:apply(M, F, A) || {M, F, A} <- MFAList], ", SampleCountName, "(Count - 1, MFAList)."])] + lists:concat([SampleCountName, "(0, _MFAList) -> ", Sep, " ok;", SampleCountName, + "(Count, MFAList) ->", Sep, " [erlang:apply(M, F, A) || {M, F, A} <- MFAList], ", + SampleCountName, "(Count - 1, MFAList)."])] }; -generate_runner(Mod, SampleCountName, Text) when is_list(Text) -> +generate_runner(Mod, SampleCountName, Text, Sep) when is_list(Text) -> case generate_text(runner, Text, true) of {0, NoDotText} -> %% very special case: embedding the text directly, without creating a new function @@ -650,9 +659,9 @@ generate_runner(Mod, SampleCountName, Text) when is_list(Text) -> fun (_Ignore) -> Mod:Mod() end, fun (SampleCount, _Ignore) -> Mod:SampleCountName(SampleCount) end, 0, 0, - [lists:concat([Mod, "() -> ", NoDotText, ", ", Mod, "()."]), - lists:concat([SampleCountName, "(0) -> ok;", SampleCountName, "(Count) -> ", - NoDotText, ", ", SampleCountName, "(Count - 1)."]), + [lists:concat([Mod, "() ->", Sep, " ", NoDotText, ",", Sep, " ", Mod, "()."]), + lists:concat([SampleCountName, "(0) ->", Sep, " ok;", Sep, SampleCountName, "(Count) ->", + Sep, " ", NoDotText, ",", Sep, " ", SampleCountName, "(Count - 1)."]), ""] }; {0, NewName, FullText} -> @@ -660,9 +669,9 @@ generate_runner(Mod, SampleCountName, Text) when is_list(Text) -> fun (_Ignore) -> Mod:Mod() end, fun (SampleCount, _Ignore) -> Mod:SampleCountName(SampleCount) end, 0, 0, - [lists:concat([Mod, "() -> ", NewName, "(), ", Mod, "()."]), - lists:concat([SampleCountName, "(0) -> ok;", SampleCountName, "(Count) -> ", - NewName, "(), ", SampleCountName, "(Count - 1)."]), + [lists:concat([Mod, "() ->", Sep, " ", NewName, "(),", Sep, " ", Mod, "()."]), + lists:concat([SampleCountName, "(0) ->", Sep, " ok;", Sep, SampleCountName, "(Count) ->", + Sep, " ", NewName, "(),", Sep, " ", SampleCountName, "(Count - 1)."]), FullText] }; {1, NewName, FullText} -> @@ -670,9 +679,9 @@ generate_runner(Mod, SampleCountName, Text) when is_list(Text) -> fun (Init) -> Mod:Mod(Init) end, fun (SampleCount, Init) -> Mod:SampleCountName(SampleCount, Init) end, 1, 1, - [lists:concat([Mod, "(Init) -> ", NewName, "(Init), ", Mod, "(Init)."]), - lists:concat([SampleCountName, "(0, _Init) -> ok;", SampleCountName, "(Count, Init) -> ", - NewName, "(Init), ", SampleCountName, "(Count - 1, Init)."]), + [lists:concat([Mod, "(Init) ->", Sep, " ", NewName, "(Init),", Sep, " ", Mod, "(Init)."]), + lists:concat([SampleCountName, "(0, _Init) ->", Sep, " ok;", Sep, SampleCountName, "(Count, Init) ->", + Sep, " ", NewName, "(Init),", Sep, " ", SampleCountName, "(Count - 1, Init).", Sep]), FullText] }; {2, NewName, FullText} -> @@ -680,14 +689,14 @@ generate_runner(Mod, SampleCountName, Text) when is_list(Text) -> fun (Init) -> Mod:Mod(Init, Init) end, fun (SampleCount, Init) -> Mod:SampleCountName(SampleCount, Init, Init) end, 2, 2, - [lists:concat([Mod, "(Init, State) -> ", Mod, "(Init, ", NewName, "(Init, State))."]), - lists:concat([SampleCountName, "(0, _Init, _State) -> ok;", SampleCountName, "(Count, Init, State) -> ", + [lists:concat([Mod, "(Init, State) ->", Sep, " ", Mod, "(Init, ", NewName, "(Init, State))."]), + lists:concat([SampleCountName, "(0, _Init, _State) ->", Sep, " ok;", Sep, SampleCountName, "(Count, Init, State) -> ", SampleCountName, "(Count - 1, Init, ", NewName, "(Init, State))."]), FullText] } end; -generate_runner(_Mod, _SampleCountName, Any) -> +generate_runner(_Mod, _SampleCountName, Any, _Sep) -> erlang:error({generate, {parse, runner, Any}}). %% generates function text diff --git a/test/erlperf_cli_SUITE.erl b/test/erlperf_cli_SUITE.erl index 7d3a08f..fbc9627 100644 --- a/test/erlperf_cli_SUITE.erl +++ b/test/erlperf_cli_SUITE.erl @@ -136,6 +136,16 @@ verbose(Config) when is_list(Config) -> Lines = filtersplit(Out, "\n"), %% TODO: actually verify that stuff printed is monitoring stuff ?assert(length(Lines) > 3), + %% expect first 5 lines to contain source code + Generated = lists:sublist(Lines, 1, 12), + ?assertEqual(Generated, [ + ">>>>>>>>>>>>>>> timer:sleep(1). ", + "-module(benchmark).", + "-export([benchmark/0, benchmark_finite/1]).","benchmark() ->", + " timer:sleep(1),"," benchmark().", + "benchmark_finite(0) ->"," ok;","benchmark_finite(Count) ->", + " timer:sleep(1),"," benchmark_finite(Count - 1).", + "<<<<<<<<<<<<<<< "]), %% parse last 2 lines [{Code, 1, C, T}] = parse_out(lists:join("\n", lists:sublist(Lines, length(Lines) - 1, 2))), ?assert(C > 250 andalso C < 1101, {qps, C}),