diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21d0f0ee..3bcf466c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,12 +18,13 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - uses: actions/checkout@v3 - - name: Build AMS + - name: Build Torch=On FAISS=On HDF5=On AMS shell: bash -l {0} run: | module load gcc/11.2.1 source /spack/share/spack/setup-env.sh spack env activate -p /ams-spack-env + rm -rf build/ mkdir build cd build export AMS_MFEM_PATH=$(spack location -i mfem) @@ -35,7 +36,6 @@ jobs: -DBUILD_SHARED_LIBS=On \ -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ -DWITH_CALIPER=On \ - -DWITH_DB=On \ -DWITH_HDF5=On \ -DWITH_EXAMPLES=On \ -DHDF5_Dir=$AMS_HDF5_PATH \ @@ -49,13 +49,115 @@ jobs: -DWITH_TORCH=On \ -DWITH_TESTS=On \ -DTorch_DIR=$AMS_TORCH_PATH \ + -DFAISS_DIR=$AMS_FAISS_PATH \ + -DWITH_AMS_DEBUG=On \ + $GITHUB_WORKSPACE + make + - name: Run tests Torch=On FAISS=On HDF5=On AMS + run: | + cd build + make test + - name: Build Torch=Off FAISS=On HDF5=On AMS + shell: bash -l {0} + run: | + module load gcc/11.2.1 + source /spack/share/spack/setup-env.sh + spack env activate -p /ams-spack-env + rm -rf build/ + mkdir build + cd build + export AMS_MFEM_PATH=$(spack location -i mfem) + export AMS_FAISS_PATH=$(spack location -i faiss) + export AMS_UMPIRE_PATH=$(spack location -i umpire) + export AMS_HDF5_PATH=$(spack location -i hdf5) + cmake \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ + -DWITH_CALIPER=On \ + -DWITH_HDF5=On \ + -DWITH_EXAMPLES=On \ + -DHDF5_Dir=$AMS_HDF5_PATH \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_CUDA=Off \ + -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ + -DMFEM_DIR=$AMS_MFEM_PATH \ -DWITH_FAISS=On \ + -DWITH_MPI=On \ + -DWITH_TORCH=Off \ + -DWITH_TESTS=On \ -DFAISS_DIR=$AMS_FAISS_PATH \ -DWITH_AMS_DEBUG=On \ - -DWITH_WORKFLOW=On \ $GITHUB_WORKSPACE make - - name: Run tests + - name: Run tests Torch=Off FAISS=On HDF5=On AMS + run: | + cd build + make test + - name: Build Torch=Off FAISS=Off HDF5=On AMS + shell: bash -l {0} + run: | + module load gcc/11.2.1 + source /spack/share/spack/setup-env.sh + spack env activate -p /ams-spack-env + rm -rf build/ + mkdir build + cd build + export AMS_MFEM_PATH=$(spack location -i mfem) + export AMS_UMPIRE_PATH=$(spack location -i umpire) + export AMS_HDF5_PATH=$(spack location -i hdf5) + cmake \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ + -DWITH_CALIPER=On \ + -DWITH_EXAMPLES=On \ + -DHDF5_Dir=$AMS_HDF5_PATH \ + -DWITH_HDF5=On \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_CUDA=Off \ + -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ + -DMFEM_DIR=$AMS_MFEM_PATH \ + -DWITH_FAISS=Off \ + -DWITH_MPI=On \ + -DWITH_TORCH=Off \ + -DWITH_TESTS=On \ + -DWITH_AMS_DEBUG=On \ + $GITHUB_WORKSPACE + make + - name: Run tests Torch=Off FAISS=Off HDF5=On AMS + run: | + cd build + make test + - name: Build Torch=Off FAISS=Off HDF5=Off AMS + shell: bash -l {0} + run: | + module load gcc/11.2.1 + source /spack/share/spack/setup-env.sh + spack env activate -p /ams-spack-env + rm -rf build/ + mkdir build + cd build + export AMS_MFEM_PATH=$(spack location -i mfem) + export AMS_UMPIRE_PATH=$(spack location -i umpire) + cmake \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ + -DWITH_CALIPER=On \ + -DWITH_EXAMPLES=On \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_CUDA=Off \ + -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ + -DMFEM_DIR=$AMS_MFEM_PATH \ + -DWITH_FAISS=Off \ + -DWITH_MPI=On \ + -DWITH_TORCH=Off \ + -DWITH_TESTS=On \ + -DWITH_AMS_DEBUG=On \ + $GITHUB_WORKSPACE + make + - name: Run tests Torch=Off FAISS=Off HDF5=Off AMS run: | cd build make test diff --git a/CMakeLists.txt b/CMakeLists.txt index 29cfbecb..dbbf5eaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,6 @@ set(AMS_EXAMPLE_INCLUDES "") option(WITH_CUDA "Option to enable CUDA" OFF) option(WITH_EXAMPLES "Build examples" OFF) option(WITH_MPI "Option to enable MPI" OFF) -option(WITH_DB "Use Database for Caching Simulation Results" OFF) option(WITH_CALIPER "Use Caliper for Profiling" OFF) option(WITH_FAISS "Use C/C++ FAISS interface for HD Cache" OFF) option(WITH_TORCH "Use C/C++ Torch interface for Surrogate Model Inference" OFF) @@ -93,77 +92,75 @@ if (WITH_AMS_DEBUG) endif() # ------------------------------------------------------------------------------ -if (WITH_DB) - list(APPEND AMS_APP_DEFINES "-D__ENABLE_DB__") +list(APPEND AMS_APP_DEFINES "-D__ENABLE_DB__") - if (WITH_REDIS) - # Temprorary fix for:= the following error which happens when using Cuda 11.6 and Redis backend - # error: #error The version of CUB in your include path is not compatible with this release of Thrust. - if (WITH_CUDA) - add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK) - endif() - if (HIREDIS_DIR) - find_path(HIREDIS_HEADER NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES include) - find_library(HIREDIS_LIB NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES lib) - else() - find_package(hiredis REQUIRED) - find_path(HIREDIS_HEADER hiredis) - find_library(HIREDIS_LIB hiredis) - endif() - message(STATUS "Hiredis library is ${HIREDIS_LIB}") - message(STATUS "Hiredis headers are ${HIREDIS_HEADER}") - list(APPEND AMS_APP_INCLUDES ${HIREDIS_HEADER}) - list(APPEND AMS_APP_LIBRARIES ${HIREDIS_LIB}) - list(APPEND AMS_APP_DEFINES "-D__ENABLE_REDIS__") - - if (REDIS_PLUS_PLUS_DIR) - find_path(REDIS_PLUS_PLUS_HEADER NAMES sw PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES include) - find_library(REDIS_PLUS_PLUS_LIB NAMES redis++ PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES lib) - else() - find_path(REDIS_PLUS_PLUS_HEADER sw) - find_library(REDIS_PLUS_PLUS_LIB redis++) - endif() - message(STATUS "Redis++ library is ${REDIS_PLUS_PLUS_LIB}") - list(APPEND AMS_APP_INCLUDES ${REDIS_PLUS_PLUS_HEADER}) - list(APPEND AMS_APP_LIBRARIES ${REDIS_PLUS_PLUS_LIB}) - endif() # WITH_REDIS - - if (WITH_HDF5) - find_package(HDF5 NAMES hdf5 COMPONENTS C shared REQUIRED NO_DEFAULT_PATH PATHS ${HDF5_Dir} ${HDF5_Dir}/share/cmake) - list(APPEND AMS_APP_INCLUDES ${HDF5_INCLUDE_DIR}) - list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) - list(APPEND AMS_APP_DEFINES "-D__ENABLE_HDF5__") - message(STATUS "HDF5 Shared Library: ${HDF5_C_SHARED_LIBRARY}") - message(STATUS "HDF5 Include directories: ${HDF5_INCLUDE_DIR}") - endif() # WITH_HDF5 +if (WITH_REDIS) + # Temprorary fix for:= the following error which happens when using Cuda 11.6 and Redis backend + # error: #error The version of CUB in your include path is not compatible with this release of Thrust. + if (WITH_CUDA) + add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK) + endif() + if (HIREDIS_DIR) + find_path(HIREDIS_HEADER NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES include) + find_library(HIREDIS_LIB NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES lib) + else() + find_package(hiredis REQUIRED) + find_path(HIREDIS_HEADER hiredis) + find_library(HIREDIS_LIB hiredis) + endif() + message(STATUS "Hiredis library is ${HIREDIS_LIB}") + message(STATUS "Hiredis headers are ${HIREDIS_HEADER}") + list(APPEND AMS_APP_INCLUDES ${HIREDIS_HEADER}) + list(APPEND AMS_APP_LIBRARIES ${HIREDIS_LIB}) + list(APPEND AMS_APP_DEFINES "-D__ENABLE_REDIS__") + + if (REDIS_PLUS_PLUS_DIR) + find_path(REDIS_PLUS_PLUS_HEADER NAMES sw PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES include) + find_library(REDIS_PLUS_PLUS_LIB NAMES redis++ PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES lib) + else() + find_path(REDIS_PLUS_PLUS_HEADER sw) + find_library(REDIS_PLUS_PLUS_LIB redis++) + endif() + message(STATUS "Redis++ library is ${REDIS_PLUS_PLUS_LIB}") + list(APPEND AMS_APP_INCLUDES ${REDIS_PLUS_PLUS_HEADER}) + list(APPEND AMS_APP_LIBRARIES ${REDIS_PLUS_PLUS_LIB}) +endif() # WITH_REDIS + +if (WITH_HDF5) + find_package(HDF5 NAMES hdf5 COMPONENTS C shared REQUIRED NO_DEFAULT_PATH PATHS ${HDF5_Dir} ${HDF5_Dir}/share/cmake) + list(APPEND AMS_APP_INCLUDES ${HDF5_INCLUDE_DIR}) + list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) + list(APPEND AMS_APP_DEFINES "-D__ENABLE_HDF5__") + message(STATUS "HDF5 Shared Library: ${HDF5_C_SHARED_LIBRARY}") + message(STATUS "HDF5 Include directories: ${HDF5_INCLUDE_DIR}") +endif() # WITH_HDF5 - if (WITH_RMQ) - if (WITH_CUDA) - add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK) - endif() - list(APPEND AMS_APP_DEFINES "-D__ENABLE_RMQ__") - - find_package(amqpcpp REQUIRED) - get_target_property(amqpcpp_INCLUDE_DIR amqpcpp INTERFACE_INCLUDE_DIRECTORIES) - list(APPEND AMS_APP_INCLUDES ${amqpcpp_INCLUDE_DIR}) - - find_package(OpenSSL REQUIRED) - if (OPENSSL_FOUND) - list(APPEND AMS_APP_INCLUDES ${OPENSSL_INCLUDE_DIR}) - list(APPEND AMS_APP_LIBRARIES "${OPENSSL_LIBRARIES}") - list(APPEND AMS_APP_LIBRARIES ssl) - message(STATUS "OpenSSL includes found: " ${OPENSSL_INCLUDE_DIR}) - message(STATUS "OpenSSL libraries found: " ${OPENSSL_LIBRARIES}) - else() - message(STATUS "OpenSSL Not Found") - endif() +if (WITH_RMQ) + if (WITH_CUDA) + add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK) + endif() + list(APPEND AMS_APP_DEFINES "-D__ENABLE_RMQ__") - find_package(libevent REQUIRED) # event loop library - list(APPEND AMS_APP_INCLUDES ${LIBEVENT_INCLUDE_DIR}) - list(APPEND AMS_APP_LIBRARIES "${LIBEVENT_LIBRARIES}") - list(APPEND AMS_APP_LIBRARIES amqpcpp event_pthreads event) - endif() # WITH_RMQ -endif() # WITH_DB + find_package(amqpcpp REQUIRED) + get_target_property(amqpcpp_INCLUDE_DIR amqpcpp INTERFACE_INCLUDE_DIRECTORIES) + list(APPEND AMS_APP_INCLUDES ${amqpcpp_INCLUDE_DIR}) + + find_package(OpenSSL REQUIRED) + if (OPENSSL_FOUND) + list(APPEND AMS_APP_INCLUDES ${OPENSSL_INCLUDE_DIR}) + list(APPEND AMS_APP_LIBRARIES "${OPENSSL_LIBRARIES}") + list(APPEND AMS_APP_LIBRARIES ssl) + message(STATUS "OpenSSL includes found: " ${OPENSSL_INCLUDE_DIR}) + message(STATUS "OpenSSL libraries found: " ${OPENSSL_LIBRARIES}) + else() + message(STATUS "OpenSSL Not Found") + endif() + + find_package(libevent REQUIRED) # event loop library + list(APPEND AMS_APP_INCLUDES ${LIBEVENT_INCLUDE_DIR}) + list(APPEND AMS_APP_LIBRARIES "${LIBEVENT_LIBRARIES}") + list(APPEND AMS_APP_LIBRARIES amqpcpp event_pthreads event) +endif() # WITH_RMQ # ------------------------------------------------------------------------------ if(NOT DEFINED UMPIRE_DIR) @@ -304,14 +301,10 @@ if (WITH_EXAMPLES) list(APPEND AMS_EXAMPLE_LIBRARIES "${MFEM_LIBRARIES}") list(APPEND AMS_EXAMPLE_LIB_DIRS "${MFEM_LIB_DIR}") - if (WITH_DB) - list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_DB__") - endif() - if (WITH_REDIS) list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_REDIS__") endif() - + if (WITH_RMQ) list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_RMQ__") list(APPEND AMS_EXAMPLE_INCLUDES ${amqpcpp_INCLUDE_DIR}) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f8ed39ca..bcc1c60c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -6,34 +6,43 @@ file(GLOB_RECURSE AMS_CURRENT_EXAMPLE_INCLUDES "*.hpp") set(AMS_EXAMPLE_SRC ${MINIAPP_INCLUDES} main.cpp) -add_executable(ams_example ${AMS_EXAMPLE_SRC} ${MINIAPP_INCLUDES}) -if (WITH_RZ) - list(APPEND AMS_EXAMPLE_SRC ${RZ_AMS_SOURCES} ${AMS_CURRENT_EXAMPLE_INCLUDES}) -endif() - -target_include_directories(ams_example PRIVATE ${AMS_EXAMPLE_INCLUDES} - ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_BINARY_DIR}/include) -target_compile_definitions(ams_example PRIVATE ${AMS_EXAMPLE_DEFINES}) -target_link_directories(ams_example PRIVATE ${AMS_EXAMPLE_LIB_DIRS}) -target_link_libraries(ams_example PUBLIC AMS ${AMS_EXAMPLE_LIBRARIES}) -if (WITH_PERFFLOWASPECT) - target_link_libraries(ams_example PUBLIC perfflowaspect_full) -endif() - -if (WITH_CUDA) - set_source_files_properties(main.cpp PROPERTIES LANGUAGE CUDA) - set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") - set_source_files_properties(app/eos_idealgas.hpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") - set_target_properties(ams_example PROPERTIES CUDA_ARCHITECTURES "${AMS_CUDA_ARCH}") - - if (WITH_RZ) - set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES LANGUAGE CUDA) - set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") - set_property(TARGET ams_example PROPERTY CUDA_SEPARABLE_COMPILATION ON) - endif() - - if (WITH_PERFFLOWASPECT) - set_property(SOURCE ${AMS_EXAMPLE_SRC} APPEND_STRING PROPERTY COMPILE_FLAGS " -ccbin clang++ -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=${PERFFLOWASPECT_LIB_DIR}/libWeavePass.so") - endif() -endif() + +function(ADDExec binary_name definitions) + if (WITH_RZ) + list(APPEND AMS_EXAMPLE_SRC ${RZ_AMS_SOURCES} ${AMS_CURRENT_EXAMPLE_INCLUDES}) + endif() + + target_include_directories(${binary_name} PRIVATE ${AMS_EXAMPLE_INCLUDES} + ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_BINARY_DIR}/include) + target_compile_definitions(${binary_name} PRIVATE ${definitions}) + target_link_directories(${binary_name} PRIVATE ${AMS_EXAMPLE_LIB_DIRS}) + target_link_libraries(${binary_name} PUBLIC AMS ${AMS_EXAMPLE_LIBRARIES}) + if (WITH_PERFFLOWASPECT) + target_link_libraries(${binary_name} PUBLIC perfflowaspect_full) + endif() + + if (WITH_CUDA) + set_source_files_properties(main.cpp PROPERTIES LANGUAGE CUDA) + set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") + set_source_files_properties(app/eos_idealgas.hpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") + set_target_properties(${binary_name} PROPERTIES CUDA_ARCHITECTURES "${AMS_CUDA_ARCH}") + + if (WITH_RZ) + set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES LANGUAGE CUDA) + set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES COMPILE_FLAGS "--expt-extended-lambda") + set_property(TARGET ${binary_name} PROPERTY CUDA_SEPARABLE_COMPILATION ON) + endif() + + if (WITH_PERFFLOWASPECT) + set_property(SOURCE ${AMS_EXAMPLE_SRC} APPEND_STRING PROPERTY COMPILE_FLAGS " -ccbin clang++ -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=${PERFFLOWASPECT_LIB_DIR}/libWeavePass.so") + endif() + endif() +endfunction() + +add_executable(no_ams_example ${AMS_EXAMPLE_SRC} ${MINIAPP_INCLUDES}) +ADDExec(no_ams_example "${AMS_EXAMPLE_DEFINES}") + +list(APPEND AMS_EXAMPLE_DEFINES "-DUSE_AMS") +add_executable(ams_example ${AMS_EXAMPLE_SRC} ${MINIAPP_INCLUDES}) +ADDExec(ams_example "${AMS_EXAMPLE_DEFINES}") diff --git a/examples/main.cpp b/examples/main.cpp index 4c6e740f..90478fc1 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -21,11 +21,8 @@ // this macro completely bypasses all AMS functionality // this allows us to check how easy is it to test ams -#define USE_AMS -#ifdef USE_AMS #include "AMS.h" -#endif using TypeValue = double; using mfem::ForallWrap; @@ -700,8 +697,9 @@ int main(int argc, char **argv) CALIPER(CALI_MARK_END("Cycle");) MPI_CALL(MPI_Barrier(MPI_COMM_WORLD)); } - +#ifdef USE_AMS delete[] workflow; +#endif // TODO: Add smart-pointers for (int mat_idx = 0; mat_idx < num_mats; ++mat_idx) { diff --git a/src/ml/hdcache.hpp b/src/ml/hdcache.hpp index cfaa9548..177c81ed 100644 --- a/src/ml/hdcache.hpp +++ b/src/ml/hdcache.hpp @@ -136,16 +136,17 @@ class HDCache #endif print(); } -#else +#else // Disabled FAISS HDCache(const std::string &cache_path, - int knbrs, bool use_device, const AMSUQPolicy uqPolicy, + int knbrs, TypeInValue threshold = 0.5) - : m_index(nullptr), + : m_index(load_cache(cache_path)), m_dim(0), - m_use_random(true), + m_use_random(false), m_knbrs(knbrs), + m_policy(uqPolicy), m_use_device(use_device), acceptable_error(threshold) { @@ -251,11 +252,13 @@ class HDCache ~HDCache() { DBG(UQModule, "Deleting UQ-Module"); +#ifdef __ENABLE_FAISS__ if (m_index) { DBG(UQModule, "Deleting HD-Cache"); m_index->reset(); delete m_index; } +#endif } //! ------------------------------------------------------------------------ @@ -626,7 +629,7 @@ class HDCache if (data_on_device) { #ifdef __ENABLE_CUDA__ - random_uq_device<<<1, 1>>>(is_acceptable, ndata, acceptable_error); + random_uq_Device<<<1, 1>>>(is_acceptable, ndata, acceptable_error); #endif } else { random_uq_host(is_acceptable, ndata, acceptable_error); diff --git a/src/ml/surrogate.hpp b/src/ml/surrogate.hpp index 8f56d6c2..16108592 100644 --- a/src/ml/surrogate.hpp +++ b/src/ml/surrogate.hpp @@ -8,12 +8,14 @@ #ifndef __AMS_SURROGATE_HPP__ #define __AMS_SURROGATE_HPP__ + #include #include #include #include #ifdef __ENABLE_TORCH__ +#include #include // One-stop header. #endif @@ -116,7 +118,9 @@ class SurrogateModel module = torch::jit::load(model_path); module.to(device); module.to(dType); - tensorOptions = torch::TensorOptions().dtype(dType).device(device); + tensorOptions = + torch::TensorOptions().dtype(dType).device(device).requires_grad( + false); } catch (const c10::Error& e) { FATAL("Error loding torch model:%s", model_path.c_str()) } @@ -179,7 +183,7 @@ class SurrogateModel inline void _evaluate(long num_elements, long num_in, size_t num_out, - TypeInValue** inputs, + const TypeInValue** inputs, TypeInValue** outputs) { } @@ -277,11 +281,19 @@ class SurrogateModel _evaluate(num_elements, inputs.size(), outputs.size(), - static_cast< const TypeInValue**>(inputs.data()), + static_cast(inputs.data()), static_cast(outputs.data())); } +#ifdef __ENABLE_TORCH__ bool is_double() { return (tensorOptions.dtype() == torch::kFloat64); } +#else + bool is_double() + { + if (typeid(TypeInValue) == typeid(double)) return true; + return false; + } +#endif bool is_cpu() { return _is_cpu; } }; diff --git a/src/wf/basedb.hpp b/src/wf/basedb.hpp index 5147c7bd..a2a95c40 100644 --- a/src/wf/basedb.hpp +++ b/src/wf/basedb.hpp @@ -679,7 +679,7 @@ class RedisDB : public BaseDB } }; -#endif // __ENABLE_HDF5__ +#endif // __ENABLE_REDIS__ #ifdef __ENABLE_RMQ__ @@ -1681,9 +1681,6 @@ BaseDB* createDB(char* dbPath, AMSDBType dbType, uint64_t rId = 0) default: return nullptr; } -#else -return nullptr; -#endif } @@ -1703,7 +1700,6 @@ std::shared_ptr> getDB(char* dbPath, { static std::unordered_map>> instances; -#ifdef __ENABLE_DB__ if (dbPath == nullptr) { std::cerr << " [WARNING] Path of DB is NULL, Please provide a valid path " "to enable db\n"; diff --git a/src/wf/debug.cpp b/src/wf/debug.cpp index ef7ec656..d2379609 100644 --- a/src/wf/debug.cpp +++ b/src/wf/debug.cpp @@ -21,6 +21,7 @@ using namespace std; */ void memUsage(double& vm_usage, double& resident_set) { + vm_usage = 0.0; resident_set = 0.0; ifstream stat_stream("/proc/self/stat", ios_base::in); //get info from proc @@ -35,8 +36,7 @@ void memUsage(double& vm_usage, double& resident_set) stime >> cutime >> cstime >> priority >> nice >> O >> itrealvalue >> starttime >> vsize >> rss; stat_stream.close(); - long page_size_kb = - sysconf(_SC_PAGE_SIZE) / 1024; // for x86-64 is configured + long page_size = sysconf(_SC_PAGE_SIZE); // for x86-64 is configured vm_usage = vsize / 1024.0; - resident_set = rss * page_size_kb; + resident_set = rss * page_size; } diff --git a/src/wf/debug.h b/src/wf/debug.h index 4a5b4153..e0765a89 100644 --- a/src/wf/debug.h +++ b/src/wf/debug.h @@ -13,16 +13,16 @@ #include -#define RED "\x1B[31m" -#define GRN "\x1B[32m" -#define YEL "\x1B[33m" -#define BLU "\x1B[34m" -#define MAG "\x1B[35m" -#define CYN "\x1B[36m" -#define WHT "\x1B[37m" +#define RED "\x1B[31m" +#define GRN "\x1B[32m" +#define YEL "\x1B[33m" +#define BLU "\x1B[34m" +#define MAG "\x1B[35m" +#define CYN "\x1B[36m" +#define WHT "\x1B[37m" #define RESET "\x1B[0m" -enum AMSVerbosity{ +enum AMSVerbosity { AMSFATAL = 0x001, AMSWARNING = 0x002, AMSINFO = 0x004, @@ -32,77 +32,91 @@ enum AMSVerbosity{ void memUsage(double& vm_usage, double& resident_set); -inline std::atomic &getInfoLevelInternal() { +inline std::atomic& getInfoLevelInternal() +{ static std::atomic InfoLevel; static std::once_flag Flag{}; std::call_once(Flag, []() { InfoLevel.store(AMSVerbosity::AMSFATAL); - if (char *EnvStr = getenv("LIBAMS_VERBOSITY_LEVEL")) - InfoLevel.store(std::stoi(EnvStr) ); // We always enable FATAL + if (char* EnvStr = getenv("LIBAMS_VERBOSITY_LEVEL")) + InfoLevel.store(std::stoi(EnvStr)); // We always enable FATAL }); return InfoLevel; } -inline uint32_t getVerbosityLevel() { return getInfoLevelInternal().load() | AMSFATAL; } +inline uint32_t getVerbosityLevel() +{ + return getInfoLevelInternal().load() | AMSFATAL; +} #define GETNAME2(name) #name #define GETNAME(name) GETNAME2(name) -#define AMSPRINTID(id) \ - do { \ - fprintf(stderr, "[ " GETNAME(id) " ] "); \ +#define AMSPRINTID(id) \ + do { \ + fprintf(stderr, "[ " GETNAME(id) " ] "); \ + } while (0); + +#define AMSPRINTMESSAGE(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ } while (0); -#define AMSPRINTMESSAGE(...) \ - do { \ - fprintf(stderr, __VA_ARGS__); \ - } while(0); - -#define AMSPRINT(id, condition, vl, color, ...) \ - if( condition && (getVerbosityLevel() & vl) ){ \ - AMSPRINTMESSAGE(color) \ - AMSPRINTID(id) \ - AMSPRINTMESSAGE(__VA_ARGS__) \ - AMSPRINTMESSAGE(RESET "\n") \ - }while(0); - -#define CFATAL(id, condition, ...) \ - AMSPRINT(id, condition, AMSVerbosity::AMSFATAL, RED, __VA_ARGS__) \ - if ( condition ){ \ - abort(); \ +#define AMSPRINT(id, condition, vl, color, ...) \ + if (condition && (getVerbosityLevel() & vl)) { \ + AMSPRINTMESSAGE(color) \ + AMSPRINTID(id) \ + AMSPRINTMESSAGE(__VA_ARGS__) \ + AMSPRINTMESSAGE(RESET "\n") \ + } \ + while (0) \ + ; + +#define CFATAL(id, condition, ...) \ + AMSPRINT(id, condition, AMSVerbosity::AMSFATAL, RED, __VA_ARGS__) \ + if (condition) { \ + abort(); \ } -#define FATAL(id, ...) \ - CFATAL(id, true, __VA_ARGS__) +#define FATAL(id, ...) CFATAL(id, true, __VA_ARGS__) #ifdef LIBAMS_VERBOSE -#define CWARNING(id, condition, ...) \ +#define CWARNING(id, condition, ...) \ AMSPRINT(id, condition, AMSVerbosity::AMSWARNING, YEL, __VA_ARGS__) -#define WARNING(id, ...) \ - CWARNING(id, true, __VA_ARGS__) +#define WARNING(id, ...) CWARNING(id, true, __VA_ARGS__) -#define CINFO(id, condition, ...) \ +#define CINFO(id, condition, ...) \ AMSPRINT(id, condition, AMSVerbosity::AMSINFO, BLU, __VA_ARGS__) -#define INFO(id, ...) \ - CINFO(id, true, __VA_ARGS__) +#define INFO(id, ...) CINFO(id, true, __VA_ARGS__) -#define CDEBUG(id, condition, ...) \ +#define CDEBUG(id, condition, ...) \ AMSPRINT(id, condition, AMSVerbosity::AMSDEBUG, MAG, __VA_ARGS__) -#define DBG(id, ...) \ - CDEBUG(id, true, __VA_ARGS__) - -#define REPORT_MEM_USAGE(id, phase) \ - do { \ - double vm, rs; \ - memUsage(vm, rs); \ - DBG(id, "Memory usage at %s is VM:%g RS:%g", phase, vm, rs); \ +#define DBG(id, ...) CDEBUG(id, true, __VA_ARGS__) + +#define REPORT_MEM_USAGE(id, phase) \ + do { \ + double vm, rs; \ + size_t watermark, current_size, actual_size; \ + memUsage(vm, rs); \ + ams::ResourceManager::getAllocatorStats(AMSResourceType::HOST, \ + watermark, \ + current_size, \ + actual_size); \ + DBG(id, \ + "Memory usage at %s is VM:%g RS:%g HWM:%lu CS:%lu AS:%lu) ", \ + phase, \ + vm, \ + rs, \ + watermark, \ + current_size, \ + actual_size); \ } while (0); -#else // LIBAMS_VERBOSE is disabled +#else // LIBAMS_VERBOSE is disabled #define CWARNING(id, condition, ...) #define WARNING(id, ...) @@ -116,6 +130,6 @@ inline uint32_t getVerbosityLevel() { return getInfoLevelInternal().load() | AMS #define DBG(id, ...) -#endif // LIBAMS_VERBOSE +#endif // LIBAMS_VERBOSE -#endif // _OMPTARGET_DEBUG_H +#endif // _OMPTARGET_DEBUG_H diff --git a/src/wf/resource_manager.hpp b/src/wf/resource_manager.hpp index b5db514e..fc2162b5 100644 --- a/src/wf/resource_manager.hpp +++ b/src/wf/resource_manager.hpp @@ -28,7 +28,6 @@ class ResourceManager { public: private: - /** @brief Used internally to allocate from the user define default resource (Device, Host Memory) */ static AMSResourceType default_resource; @@ -137,15 +136,23 @@ class ResourceManager * @return Pointer to allocated elements. */ template -PERFFASPECT() - static TypeInValue* allocate(size_t nvalues, AMSResourceType dev = default_resource) + PERFFASPECT() + static TypeInValue* allocate(size_t nvalues, + AMSResourceType dev = default_resource) { static auto& rm = umpire::ResourceManager::getInstance(); - DBG(ResourceManager, "Requesting to allocate %ld values using allocator :%s", nvalues, getAllocatorName(dev)); + DBG(ResourceManager, + "Requesting to allocate %ld values using allocator :%s", + nvalues, + getAllocatorName(dev)); auto alloc = rm.getAllocator(allocator_ids[dev]); - TypeInValue *ret = static_cast(alloc.allocate(nvalues * sizeof(TypeInValue))); - CFATAL(ResourceManager, ret == nullptr, - "Failed to allocated %ld values on device %d", nvalues, dev); + TypeInValue* ret = static_cast( + alloc.allocate(nvalues * sizeof(TypeInValue))); + CFATAL(ResourceManager, + ret == nullptr, + "Failed to allocated %ld values on device %d", + nvalues, + dev); return ret; } @@ -156,7 +163,7 @@ PERFFASPECT() * @return void. */ template -PERFFASPECT() + PERFFASPECT() static void deallocate(TypeInValue* data, AMSResourceType dev) { static auto& rm = umpire::ResourceManager::getInstance(); @@ -171,7 +178,7 @@ PERFFASPECT() * @param[in] dev resource to register the memory to. * @return void. */ -PERFFASPECT() + PERFFASPECT() static void registerExternal(void* ptr, size_t nBytes, AMSResourceType dev = default_resource) @@ -201,7 +208,7 @@ PERFFASPECT() * @return void. */ template -PERFFASPECT() + PERFFASPECT() static void copy(TypeInValue* src, TypeInValue* dest, size_t size = 0) { static auto& rm = umpire::ResourceManager::getInstance(); @@ -219,6 +226,26 @@ PERFFASPECT() for (auto* I : dPtr) deallocate(I); } + + /** @brief Returns the memory consumption of the given resource as measured from Umpire. + * @param[in] resource The memory pool to get the consumption from. + * @param[out] wm the highest memory allocation that umpire has performed until now. + * @param[out] cs The current size of the pool. This can be smaller than the actual size. + * @param[out] as The actual size of the pool.. + * @return void. + */ + static void getAllocatorStats(AMSResourceType resource, + size_t& wm, + size_t& cs, + size_t& as) + { + auto& rm = umpire::ResourceManager::getInstance(); + auto alloc = rm.getAllocator(allocator_ids[resource]); + wm = alloc.getHighWatermark(); + cs = alloc.getCurrentSize(); + as = alloc.getActualSize(); + return; + } //! ------------------------------------------------------------------------ }; diff --git a/src/wf/workflow.hpp b/src/wf/workflow.hpp index 02ec40e1..bf69ab52 100644 --- a/src/wf/workflow.hpp +++ b/src/wf/workflow.hpp @@ -107,7 +107,7 @@ class AMSWorkflow const int numOut = outputs.size(); // No database, so just de-allocate and return - if (DB) return; + if (!DB) return; std::vector hInputs, hOutputs; @@ -229,10 +229,7 @@ class AMSWorkflow void set_hdcache(HDCache *_hdcache) { hdcache = _hdcache; } - ~AMSWorkflow() - { - DBG(Workflow, "Destroying Workflow Handler"); - } + ~AMSWorkflow() { DBG(Workflow, "Destroying Workflow Handler"); } /** @brief This is the main entry point of AMSLib and replaces the original