Ats5

BUILD.md

@@ -0,0 +1,3 @@
+```
+CALIPER_ROOT_DIR=<path>/Caliper-Adiak/install CC=cc CXX=CC cmake   -Dadiak_DIR=<path>/Adiak/install/lib/cmake/adiak/ -Dcaliper_DIR=<path>/Caliper-Adiak/install/share/cmake/caliper/  -DMETIS_ROOT_DIR==<path>/new/metis/install -DCMAKE_BUILD_TYPE=Release  -DWITH_ADIAK=ON  ../src 
+```

src/config.h.in

@@ -62,6 +62,7 @@
 #endif
 
 #ifdef caliper_FOUND
+#include <adiak.h>
 #include <caliper/cali.h>
 #endif
 
@@ -97,6 +98,11 @@ inline void Insist(const bool pass, const std::string &message) {
 
 #ifdef caliper_FOUND
 
+void cali_init() { 
+     MPI_Comm comm = MPI_COMM_WORLD;
+     adiak_init(&comm);
+     adiak_collect_all();
+}
 
 void wrapped_cali_mark_begin(const char *timer_name) {
     CALI_MARK_BEGIN(timer_name);
@@ -106,19 +112,19 @@ void wrapped_cali_mark_end(const char *timer_name) {
 }
 
 #else
-void wrapped_cali_mark_begin(const char *timer_name) {
+void cali_init() { 
   int rank;
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   if (rank == 0) {
-    std::cout<<"Branson built without caliper, not starting timer: "<<timer_name<<std::endl;
+    std::cout<<"Branson built without caliper"<<std::endl;
   }
 }
-void wrapped_cali_mark_end(const char *timer_name) {
-  int rank;
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  if (rank == 0) {
-    std::cout<<"Branson built without caliper, not ending timer: "<<timer_name<<std::endl;
-  }
+// Rely upon compiler optimizing this inline function out 
+inline void wrapped_cali_mark_begin(const char *timer_name) {
+  return;
+}
+inline void wrapped_cali_mark_end(const char *timer_name) {
+  return;
 }
 
 #endif

src/config/find_tpls.cmake

@@ -106,7 +106,7 @@ macro(setupTPLs)
     endif()
 
     message( STATUS "Looking for caliper..." )
-    find_package( caliper QUIET)
+    find_package( caliper )
     if( caliper_FOUND )
       message( STATUS "Looking for caliper.....found ${CALIPER_LIBRARY}" )
     else()
@@ -120,6 +120,21 @@ macro(setupTPLs)
       PURPOSE "Code instrumentation for performance analysis"
    )
 
+   if (WITH_ADIAK)
+     find_package(adiak)
+     if (adiak_FOUND)
+      message( STATUS "Looking for adiak.....found ${adiak_LIBRARIES}" )
+      include_directories("${adiak_INCLUDE_DIR}")
+     else()
+       message(WARNING "Adiak support was requested but Adiak was not found")
+     endif()
+     set_package_properties( adiak PROPERTIES
+      DESCRIPTION "ADIAK"
+      TYPE OPTIONAL
+      URL "http://software.llnl.gov/Adiak/"
+      PURPOSE "Metadata for perf data.")
+
+   endif()
   endif()
 
   ##############################################################################

src/imc_state.h

@@ -17,13 +17,14 @@
 #include <iostream>
 #include <mpi.h>
 #include <vector>
-
+#include "config.h" 
 #include "RNG.h"
 #include "constants.h"
 #include "input.h"
 #include "message_counter.h"
 #include "photon.h"
 #include "cell.h"
+
 #include <iomanip>
 
 //==============================================================================
@@ -295,6 +296,7 @@ class IMC_State {
     m_step++;
   }
 
+  //  wrapped_cali_mark_begin("transport diagnostics");
   //! Set pre-transport census energy  (diagnostic)
   void set_pre_census_E(double _pre_census_E) { pre_census_E = _pre_census_E; }
 
@@ -340,6 +342,7 @@ class IMC_State {
     step_receives_posted = mctr.n_receives_posted;
     step_receives_completed = mctr.n_receives_completed;
   }
+  //  wrapped_cali_mark_end("transport diagnostics");
 
   //! Set the number of cells requested in mesh passing method this timestep
   void set_step_cells_requested(uint64_t _step_cells_requested) {

src/main.cc

@@ -36,7 +36,7 @@ using std::vector;
 
 int main(int argc, char **argv) {
   MPI_Init(&argc, &argv);
-
+  cali_init();
   // check to see if number of arguments is correct
   if (argc != 2) {
     cout << "Usage: BRANSON <path_to_input_file>" << endl;
@@ -103,11 +103,15 @@ int main(int argc, char **argv) {
 
     timers.start_timer("Total non-setup");
 
-    if (input.get_dd_mode() == PARTICLE_PASS)
+    if (input.get_dd_mode() == PARTICLE_PASS) {
+      wrapped_cali_mark_begin("imc particle pass driver");
       imc_particle_pass_driver(mesh, imc_state, imc_p, mpi_types, mpi_info);
-    else if (input.get_dd_mode() == REPLICATED)
+      wrapped_cali_mark_end("imc particle pass driver");
+    } else if (input.get_dd_mode() == REPLICATED) {
+      wrapped_cali_mark_begin("imc replicated driver");
       imc_replicated_driver(mesh, imc_state, imc_p, mpi_types, mpi_info);
-    else {
+      wrapped_cali_mark_end("imc replicated driver");
+    } else {
       cout << "Driver for DD transport method currently not supported" << endl;
       exit(EXIT_FAILURE);
     }

src/particle_pass_driver.h

@@ -64,39 +64,56 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state,
     mctr.reset_counters();
 
     //set opacity, Fleck factor, all energy to source
+    wrapped_cali_mark_begin("calculate photon energy");
     mesh.calculate_photon_energy(imc_state, n_user_photons);
+    wrapped_cali_mark_end("calculate photon energy");
 
     // all reduce to get total source energy to make correct number of
     // particles on each rank
     double global_source_energy = mesh.get_total_photon_E();
     MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM,
                   MPI_COMM_WORLD);
 
+    wrapped_cali_mark_begin("calculate pre census energy");
     imc_state.set_pre_census_E(get_photon_list_E(census_photons));
+    wrapped_cali_mark_end("calculate pre census energy");
 
     // make gpu setup object, may want to source on GPU later so make it before sourcing here
     GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells());
 
     // setup source
-    if (imc_state.get_step() == 1)
+    if (imc_state.get_step() == 1) {
+      wrapped_cali_mark_begin("make initial census photons");
       census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy);
+      wrapped_cali_mark_end("make initial census photons");
+    }
 
     imc_state.set_pre_census_E(get_photon_list_E(census_photons));
     MPI_Barrier(MPI_COMM_WORLD);
     // make emission and source photons
+    wrapped_cali_mark_begin("make emission and source photons");
     auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy);
+    wrapped_cali_mark_end("make emission and source photons");
     // add the census photons
+    wrapped_cali_mark_begin("add the census photons");
     all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end());
+    wrapped_cali_mark_end("add the census photons");
 
+    wrapped_cali_mark_begin("set transported particles");
     imc_state.set_transported_particles(all_photons.size());
-
+    wrapped_cali_mark_end("set transported particles");
+
     imc_state.print_memory_estimate(rank, n_ranks, mesh.get_n_local_cells(), all_photons.size());
 
     // add barrier here to make sure the transport timer starts at roughly the same time
     MPI_Barrier(MPI_COMM_WORLD);
+    wrapped_cali_mark_begin("transport photons");
     census_photons = particle_pass_transport(mesh, gpu_setup, imc_parameters, mpi_info, mpi_types, imc_state, mctr, abs_E, track_E, all_photons, imc_parameters.get_n_omp_threads());
+    wrapped_cali_mark_end("transport photons");
 
+    wrapped_cali_mark_begin("update temperature");
     mesh.update_temperature(abs_E, track_E, imc_state);
+    wrapped_cali_mark_end("update temperature");
 
     // update time for next step
     imc_state.print_conservation(imc_parameters.get_dd_mode());
@@ -118,7 +135,10 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state,
     mem_record.read_meminfo(meminfo_string);
 #endif
 
+    wrapped_cali_mark_begin("update time for next step");
     imc_state.next_time_step();
+    wrapped_cali_mark_end("update time for next step");
+
   }
 
 #ifdef USE_MEMORY_RECORD

src/replicated_driver.h

@@ -50,56 +50,76 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state,
     mctr.reset_counters();
 
     // set opacity, Fleck factor, all energy to source
+    wrapped_cali_mark_begin("calculate photon energy");
     mesh.calculate_photon_energy(imc_state, n_user_photons);
+    wrapped_cali_mark_end("calculate photon energy");
 
     // all reduce to get total source energy to make correct number of articles on each rank
     double global_source_energy = mesh.get_total_photon_E();
     MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM,
                   MPI_COMM_WORLD);
-
+    wrapped_cali_mark_begin("calculate pre census energy");
     imc_state.set_pre_census_E(get_photon_list_E(census_photons));
+    wrapped_cali_mark_end("calculate pre census energy");
 
     // make gpu setup object, may want to source on GPU later so make it before sourcing here
     GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells());
 
     // setup source
     Timer t_source;
     t_source.start_timer("source");
-    if (imc_state.get_step() == 1)
+    if (imc_state.get_step() == 1) { 
+      wrapped_cali_mark_begin("make initial census photons");
       census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy);
+      wrapped_cali_mark_end("make initial census photons");
+    }
     imc_state.set_pre_census_E(get_photon_list_E(census_photons));
     // make emission and source photons
+    wrapped_cali_mark_begin("make emission and source photons");
     auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy);
+    wrapped_cali_mark_end("make emission and source photons");
     // add the census photons
+    wrapped_cali_mark_begin("add the census photons");
     all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end());
+    wrapped_cali_mark_end("add the census photons");
+
     t_source.stop_timer("source");
     if (rank ==0)
       std::cout<<"source time: "<<t_source.get_time("source")<<std::endl;
-
+
+    wrapped_cali_mark_begin("set transported particles");
     imc_state.set_transported_particles(all_photons.size());
+    wrapped_cali_mark_end("set transported particles");
 
     imc_state.print_memory_estimate(rank, n_ranks,  mesh.get_n_local_cells(), all_photons.size());
 
     // add barrier here to make sure the transport timer starts at roughly the same time
     MPI_Barrier(MPI_COMM_WORLD);
 
+    wrapped_cali_mark_begin("transport photons");
     census_photons =
         replicated_transport(mesh, gpu_setup, imc_state, abs_E, track_E, all_photons, imc_parameters.get_n_omp_threads());
+    wrapped_cali_mark_end("transport photons");
 
     // reduce the abs_E and the track weighted energy (for T_r)
     MPI_Allreduce(MPI_IN_PLACE, &abs_E[0], mesh.get_n_global_cells(),
                   MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
     MPI_Allreduce(MPI_IN_PLACE, &track_E[0], mesh.get_n_global_cells(),
                   MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
 
+    wrapped_cali_mark_begin("update temperature");
     mesh.update_temperature(abs_E, track_E, imc_state);
+    wrapped_cali_mark_end("update temperature");
 
     MPI_Barrier(MPI_COMM_WORLD);
     // for replicated, just let root do conservation
     if (rank) {
+      wrapped_cali_mark_begin("root doing conversion");
       imc_state.set_absorbed_E(0.0);
       imc_state.set_pre_mat_E(0.0);
       imc_state.set_post_mat_E(0.0);
+      wrapped_cali_mark_end("root doing conversion");
+
     }
 
     imc_state.print_conservation(imc_parameters.get_dd_mode());
@@ -116,7 +136,10 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state,
     }
 
     // update time for next step
+    wrapped_cali_mark_begin("update time for next step");
     imc_state.next_time_step();
+    wrapped_cali_mark_end("update time for next step");
+
   }
 }
 

src/replicated_transport.h

@@ -66,16 +66,24 @@ std::vector<Photon> replicated_transport(
   uint32_t rank_cell_offset{0}; // no offset in replicated mesh
   if(gpu_setup.use_gpu_transporter() && gpu_available ) {
     t_transport.start_timer("gpu transport");
+    wrapped_cali_mark_begin("gpu transport photons");
     gpu_transport_photons(rank_cell_offset, all_photons, gpu_setup.get_device_cells_ptr(), cell_tallies);
+    wrapped_cali_mark_end("gpu transport photons");
     t_transport.stop_timer("gpu transport");
+
     std::cout<<"gpu transport time: "<<t_transport.get_time("gpu transport")<<std::endl;
   }
   else {
+    wrapped_cali_mark_begin("cpu transport photons");
     cpu_transport_photons(rank_cell_offset, all_photons, mesh.get_cells(), cell_tallies, n_omp_threads);
+    wrapped_cali_mark_end("cpu transport photons");
+
   }
 
   // post process photons, account for escaped energy and add particles to census
+  wrapped_cali_mark_begin("post process photons");
   post_process_photons(next_dt, all_photons, census_list, census_E, exit_E);
+  wrapped_cali_mark_end("post process photons");
 
   // copy cell tallies back out to rank_abs_E and rank_track_E
   double total_abs = 0;