From 94823ae047d1deb37ba42e5e90b9bfe741fc2158 Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Thu, 8 Feb 2024 13:50:48 -0800 Subject: [PATCH 1/9] add more caliper goodness --- src/config.h.in | 22 ++++++++++++++-------- src/imc_state.h | 2 ++ src/main.cc | 7 ++++++- src/replicated_driver.h | 27 +++++++++++++++++++++++++-- src/replicated_transport.h | 8 ++++++++ 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/config.h.in b/src/config.h.in index 39f0f57..6516ada 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -97,6 +97,12 @@ inline void Insist(const bool pass, const std::string &message) { #ifdef caliper_FOUND +void cali_init() { + adiak_init(&comm); + adiak_collect_all(); + //adiak_namevalue("compiler", adiak_general, NULL, "%s", AMG2023_COMPILER_ID); + //adiak_namevalue("compiler version", adiak_general, NULL, "%s", AMG2023_COMPILER_VERSION); +} void wrapped_cali_mark_begin(const char *timer_name) { CALI_MARK_BEGIN(timer_name); @@ -106,19 +112,19 @@ void wrapped_cali_mark_end(const char *timer_name) { } #else -void wrapped_cali_mark_begin(const char *timer_name) { +void cali_init() { int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (rank == 0) { - std::cout<<"Branson built without caliper, not starting timer: "<" << endl; @@ -104,9 +104,14 @@ int main(int argc, char **argv) { timers.start_timer("Total non-setup"); if (input.get_dd_mode() == PARTICLE_PASS) + wrapped_cali_mark_begin("imc particle pass driver"); imc_particle_pass_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); + wrapped_cali_mark_end("imc particle pass driver"); + else if (input.get_dd_mode() == REPLICATED) + wrapped_cali_mark_begin("imc replicated driver"); imc_replicated_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); + wrapped_cali_mark_end("imc replicated driver"); else { cout << "Driver for DD transport method currently not supported" << endl; exit(EXIT_FAILURE); diff --git a/src/replicated_driver.h b/src/replicated_driver.h index 1412ecb..1fd8c63 100644 --- a/src/replicated_driver.h +++ b/src/replicated_driver.h @@ -50,14 +50,17 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state, mctr.reset_counters(); // set opacity, Fleck factor, all energy to source + wrapped_cali_mark_begin("calculate photon energy"); mesh.calculate_photon_energy(imc_state, n_user_photons); + wrapped_cali_mark_end("calculate photon energy"); // all reduce to get total source energy to make correct number of articles on each rank double global_source_energy = mesh.get_total_photon_E(); MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - + wrapped_cali_mark_begin("calculate pre census energy"); imc_state.set_pre_census_E(get_photon_list_E(census_photons)); + wrapped_cali_mark_end("calculate pre census energy"); // make gpu setup object, may want to source on GPU later so make it before sourcing here GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells()); @@ -66,25 +69,37 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state, Timer t_source; t_source.start_timer("source"); if (imc_state.get_step() == 1) + wrapped_cali_mark_begin("make initial census photons"); census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make initial census photons"); + imc_state.set_pre_census_E(get_photon_list_E(census_photons)); // make emission and source photons + wrapped_cali_mark_begin("make emission and source photons"); auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make emission and source photons"); // add the census photons + wrapped_cali_mark_begin("add the census photons"); all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end()); + wrapped_cali_mark_end("add the census photons"); + t_source.stop_timer("source"); if (rank ==0) std::cout<<"source time: "< replicated_transport( uint32_t rank_cell_offset{0}; // no offset in replicated mesh if(gpu_setup.use_gpu_transporter() && gpu_available ) { t_transport.start_timer("gpu transport"); + wrapped_cali_mark_begin("gpu transport photons"); gpu_transport_photons(rank_cell_offset, all_photons, gpu_setup.get_device_cells_ptr(), cell_tallies); + wrapped_cali_mark_end("gpu transport photons"); t_transport.stop_timer("gpu transport"); + std::cout<<"gpu transport time: "< Date: Thu, 8 Feb 2024 14:55:52 -0800 Subject: [PATCH 2/9] init cali --- src/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 739e6c1..cbcbbf5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -36,7 +36,7 @@ using std::vector; int main(int argc, char **argv) { MPI_Init(&argc, &argv); - + cali_init(); // check to see if number of arguments is correct if (argc != 2) { cout << "Usage: BRANSON " << endl; From d62af95b999ddac8d2faf66e6fc4c6c1630e604e Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Fri, 9 Feb 2024 12:54:21 -0800 Subject: [PATCH 3/9] use MPI_COMM_WORLD --- src/config.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.h.in b/src/config.h.in index 6516ada..1c85211 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -98,7 +98,7 @@ inline void Insist(const bool pass, const std::string &message) { #ifdef caliper_FOUND void cali_init() { - adiak_init(&comm); + adiak_init(MPI_COMM_WORLD); adiak_collect_all(); //adiak_namevalue("compiler", adiak_general, NULL, "%s", AMG2023_COMPILER_ID); //adiak_namevalue("compiler version", adiak_general, NULL, "%s", AMG2023_COMPILER_VERSION); From 395377668fbe84ba64ede6679c04c90ce9a1ce17 Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Fri, 9 Feb 2024 16:14:46 -0700 Subject: [PATCH 4/9] update to fix init and usage of caliper --- inputs/3D_hohlraum_single_node.xml | 4 ++-- src/config.h.in | 4 +++- src/config/find_tpls.cmake | 17 ++++++++++++++++- src/imc_state.h | 7 ++++--- src/main.cc | 7 +++---- src/replicated_driver.h | 4 ++-- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/inputs/3D_hohlraum_single_node.xml b/inputs/3D_hohlraum_single_node.xml index b10eacd..2f88b23 100644 --- a/inputs/3D_hohlraum_single_node.xml +++ b/inputs/3D_hohlraum_single_node.xml @@ -6,11 +6,11 @@ 0.01 1.0 1.0 - 10000000 + 100000 14706 FALSE FALSE - TRUE + FALSE REPLICATED 1 1 diff --git a/src/config.h.in b/src/config.h.in index 1c85211..5e9c0ae 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -62,6 +62,7 @@ #endif #ifdef caliper_FOUND +#include #include #endif @@ -98,7 +99,8 @@ inline void Insist(const bool pass, const std::string &message) { #ifdef caliper_FOUND void cali_init() { - adiak_init(MPI_COMM_WORLD); + MPI_Comm comm = MPI_COMM_WORLD; + adiak_init(&comm); adiak_collect_all(); //adiak_namevalue("compiler", adiak_general, NULL, "%s", AMG2023_COMPILER_ID); //adiak_namevalue("compiler version", adiak_general, NULL, "%s", AMG2023_COMPILER_VERSION); diff --git a/src/config/find_tpls.cmake b/src/config/find_tpls.cmake index 22c2ea9..c03dd34 100644 --- a/src/config/find_tpls.cmake +++ b/src/config/find_tpls.cmake @@ -106,7 +106,7 @@ macro(setupTPLs) endif() message( STATUS "Looking for caliper..." ) - find_package( caliper QUIET) + find_package( caliper ) if( caliper_FOUND ) message( STATUS "Looking for caliper.....found ${CALIPER_LIBRARY}" ) else() @@ -120,6 +120,21 @@ macro(setupTPLs) PURPOSE "Code instrumentation for performance analysis" ) + if (WITH_ADIAK) + find_package(adiak) + if (adiak_FOUND) + message( STATUS "Looking for adiak.....found ${adiak_LIBRARIES}" ) + include_directories("${adiak_INCLUDE_DIR}") + else() + message(WARNING "Adiak support was requested but Adiak was not found") + endif() + set_package_properties( adiak PROPERTIES + DESCRIPTION "ADIAK" + TYPE OPTIONAL + URL "http://software.llnl.gov/Adiak/" + PURPOSE "Metadata for perf data.") + + endif() endif() ############################################################################## diff --git a/src/imc_state.h b/src/imc_state.h index 590a9df..2ede3f6 100644 --- a/src/imc_state.h +++ b/src/imc_state.h @@ -17,13 +17,14 @@ #include #include #include - +#include "config.h" #include "RNG.h" #include "constants.h" #include "input.h" #include "message_counter.h" #include "photon.h" #include "cell.h" + #include //============================================================================== @@ -295,7 +296,7 @@ class IMC_State { m_step++; } - wrapped_cali_mark_begin("transport diagnostics"); + // wrapped_cali_mark_begin("transport diagnostics"); //! Set pre-transport census energy (diagnostic) void set_pre_census_E(double _pre_census_E) { pre_census_E = _pre_census_E; } @@ -341,7 +342,7 @@ class IMC_State { step_receives_posted = mctr.n_receives_posted; step_receives_completed = mctr.n_receives_completed; } - wrapped_cali_mark_end("transport diagnostics"); + // wrapped_cali_mark_end("transport diagnostics"); //! Set the number of cells requested in mesh passing method this timestep void set_step_cells_requested(uint64_t _step_cells_requested) { diff --git a/src/main.cc b/src/main.cc index cbcbbf5..48e34cd 100644 --- a/src/main.cc +++ b/src/main.cc @@ -103,16 +103,15 @@ int main(int argc, char **argv) { timers.start_timer("Total non-setup"); - if (input.get_dd_mode() == PARTICLE_PASS) + if (input.get_dd_mode() == PARTICLE_PASS) { wrapped_cali_mark_begin("imc particle pass driver"); imc_particle_pass_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); wrapped_cali_mark_end("imc particle pass driver"); - - else if (input.get_dd_mode() == REPLICATED) + } else if (input.get_dd_mode() == REPLICATED) { wrapped_cali_mark_begin("imc replicated driver"); imc_replicated_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); wrapped_cali_mark_end("imc replicated driver"); - else { + } else { cout << "Driver for DD transport method currently not supported" << endl; exit(EXIT_FAILURE); } diff --git a/src/replicated_driver.h b/src/replicated_driver.h index 1fd8c63..8e45f31 100644 --- a/src/replicated_driver.h +++ b/src/replicated_driver.h @@ -68,11 +68,11 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state, // setup source Timer t_source; t_source.start_timer("source"); - if (imc_state.get_step() == 1) + if (imc_state.get_step() == 1) { wrapped_cali_mark_begin("make initial census photons"); census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy); wrapped_cali_mark_end("make initial census photons"); - + } imc_state.set_pre_census_E(get_photon_list_E(census_photons)); // make emission and source photons wrapped_cali_mark_begin("make emission and source photons"); From cc6e832fdd52bc93a36044f68f22df405c85e2ea Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Thu, 29 Feb 2024 17:45:39 -0700 Subject: [PATCH 5/9] adiak --- src/config.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.h.in b/src/config.h.in index 5e9c0ae..f617dca 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -101,7 +101,7 @@ inline void Insist(const bool pass, const std::string &message) { void cali_init() { MPI_Comm comm = MPI_COMM_WORLD; adiak_init(&comm); - adiak_collect_all(); + adiak_collect_all(); //adiak_namevalue("compiler", adiak_general, NULL, "%s", AMG2023_COMPILER_ID); //adiak_namevalue("compiler version", adiak_general, NULL, "%s", AMG2023_COMPILER_VERSION); } From 6b2c57452b015712bd1dfbae4672c50c6081e5b5 Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Mon, 25 Mar 2024 17:05:50 -0600 Subject: [PATCH 6/9] revert changes to input --- inputs/3D_hohlraum_single_node.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inputs/3D_hohlraum_single_node.xml b/inputs/3D_hohlraum_single_node.xml index 2f88b23..b10eacd 100644 --- a/inputs/3D_hohlraum_single_node.xml +++ b/inputs/3D_hohlraum_single_node.xml @@ -6,11 +6,11 @@ 0.01 1.0 1.0 - 100000 + 10000000 14706 FALSE FALSE - FALSE + TRUE REPLICATED 1 1 From 52582d22a7c185903681b4725609af18c768c8fe Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Mon, 25 Mar 2024 17:07:12 -0600 Subject: [PATCH 7/9] remove unused adiak calls --- src/config.h.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/config.h.in b/src/config.h.in index f617dca..0ab21ec 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -102,8 +102,6 @@ void cali_init() { MPI_Comm comm = MPI_COMM_WORLD; adiak_init(&comm); adiak_collect_all(); - //adiak_namevalue("compiler", adiak_general, NULL, "%s", AMG2023_COMPILER_ID); - //adiak_namevalue("compiler version", adiak_general, NULL, "%s", AMG2023_COMPILER_VERSION); } void wrapped_cali_mark_begin(const char *timer_name) { From 3015841ed218218fb1a505f050cb603a61328ec0 Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Mon, 25 Mar 2024 17:34:19 -0600 Subject: [PATCH 8/9] add caliper to particle pass driver --- src/particle_pass_driver.h | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/particle_pass_driver.h b/src/particle_pass_driver.h index 2c74921..81fafd8 100644 --- a/src/particle_pass_driver.h +++ b/src/particle_pass_driver.h @@ -64,7 +64,9 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, mctr.reset_counters(); //set opacity, Fleck factor, all energy to source + wrapped_cali_mark_begin("calculate photon energy"); mesh.calculate_photon_energy(imc_state, n_user_photons); + wrapped_cali_mark_end("calculate photon energy"); // all reduce to get total source energy to make correct number of // particles on each rank @@ -72,31 +74,46 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + wrapped_cali_mark_begin("calculate pre census energy"); imc_state.set_pre_census_E(get_photon_list_E(census_photons)); + wrapped_cali_mark_end("calculate pre census energy"); // make gpu setup object, may want to source on GPU later so make it before sourcing here GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells()); // setup source - if (imc_state.get_step() == 1) + if (imc_state.get_step() == 1) { + wrapped_cali_mark_begin("make initial census photons"); census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make initial census photons"); + } imc_state.set_pre_census_E(get_photon_list_E(census_photons)); MPI_Barrier(MPI_COMM_WORLD); // make emission and source photons + wrapped_cali_mark_begin("make emission and source photons"); auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make emission and source photons"); // add the census photons + wrapped_cali_mark_begin("add the census photons"); all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end()); + wrapped_cali_mark_end("add the census photons"); + wrapped_cali_mark_begin("set transported particles"); imc_state.set_transported_particles(all_photons.size()); - + wrapped_cali_mark_end("set transported particles"); + imc_state.print_memory_estimate(rank, n_ranks, mesh.get_n_local_cells(), all_photons.size()); // add barrier here to make sure the transport timer starts at roughly the same time MPI_Barrier(MPI_COMM_WORLD); + wrapped_cali_mark_begin("transport photons"); census_photons = particle_pass_transport(mesh, gpu_setup, imc_parameters, mpi_info, mpi_types, imc_state, mctr, abs_E, track_E, all_photons, imc_parameters.get_n_omp_threads()); + wrapped_cali_mark_end("transport photons"); + wrapped_cali_mark_begin("update temperature"); mesh.update_temperature(abs_E, track_E, imc_state); + wrapped_cali_mark_end("update temperature"); // update time for next step imc_state.print_conservation(imc_parameters.get_dd_mode()); @@ -118,7 +135,10 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, mem_record.read_meminfo(meminfo_string); #endif + wrapped_cali_mark_begin("update time for next step"); imc_state.next_time_step(); + wrapped_cali_mark_end("update time for next step"); + } #ifdef USE_MEMORY_RECORD From db22eb88726c56a4bf5672cf908b760eda75c4aa Mon Sep 17 00:00:00 2001 From: "Galen M. Shipman" Date: Mon, 25 Mar 2024 17:52:15 -0600 Subject: [PATCH 9/9] add example of how to build with caliper --- BUILD.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 BUILD.md diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000..348685b --- /dev/null +++ b/BUILD.md @@ -0,0 +1,3 @@ +``` +CALIPER_ROOT_DIR=/Caliper-Adiak/install CC=cc CXX=CC cmake -Dadiak_DIR=/Adiak/install/lib/cmake/adiak/ -Dcaliper_DIR=/Caliper-Adiak/install/share/cmake/caliper/ -DMETIS_ROOT_DIR==/new/metis/install -DCMAKE_BUILD_TYPE=Release -DWITH_ADIAK=ON ../src +```