Merge branch 'main' into sundials_work

.github/workflows/dependencies/dependencies.sh

@@ -17,4 +17,5 @@ sudo apt-get install -y --no-install-recommends\
     python3         \
     python3-pip
 
-python3 -m pip install -U pip setuptools wheel
+python3 -m pip install -U pip
+python3 -m pip install -U build packaging setuptools wheel

.github/workflows/dependencies/dependencies_clang6.sh

@@ -15,4 +15,5 @@ sudo apt-get install -y  \
     python3              \
     python3-pip
 
-python3 -m pip install -U pip setuptools wheel
+python3 -m pip install -U pip
+python3 -m pip install -U build packaging setuptools wheel

.github/workflows/dependencies/dependencies_gcc10.sh

@@ -18,4 +18,5 @@ sudo apt-get install -y --no-install-recommends \
     python3            \
     python3-pip
 
-python3 -m pip install -U pip setuptools wheel
+python3 -m pip install -U pip
+python3 -m pip install -U build packaging setuptools wheel

.github/workflows/dependencies/dependencies_nofortran.sh

@@ -21,4 +21,5 @@ sudo apt-get install -y --no-install-recommends\
     python3         \
     python3-pip
 
-python3 -m pip install -U pip setuptools wheel
+python3 -m pip install -U pip
+python3 -m pip install -U build packaging setuptools wheel

ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt

@@ -0,0 +1,15 @@
+if (AMReX_SPACEDIM EQUAL 1)
+   return()
+endif ()
+
+# List of source files
+set(_sources main.cpp myfunc.H)
+list(TRANSFORM _sources PREPEND "Source/")
+
+# List of input files
+file( GLOB_RECURSE _input_files LIST_DIRECTORIES false Exec/input* )
+
+setup_tutorial(_sources _input_files)
+
+unset( _sources )
+unset( _input_files   )

ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile

@@ -0,0 +1,51 @@
+# AMREX_HOME defines the directory in which we will find all the AMReX code.
+AMREX_HOME ?= ../../../../../amrex
+
+DEBUG        = FALSE
+USE_MPI      = TRUE
+USE_OMP      = FALSE
+COMP         = gnu
+DIM          = 3
+USE_RPATH    = TRUE
+USE_SUNDIALS = TRUE
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.defs
+
+include ../Source/Make.package
+VPATH_LOCATIONS  += ../Source
+INCLUDE_LOCATIONS += ../Source
+
+ifeq ($(USE_SUNDIALS),TRUE)
+  ifeq ($(USE_CUDA),TRUE)
+    SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir_cuda
+  else
+    SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir
+  endif
+  ifeq ($(NERSC_HOST),perlmutter)
+    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib64
+  else
+    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib
+  endif
+
+  USE_CVODE_LIBS ?= TRUE
+  USE_ARKODE_LIBS ?= TRUE
+
+  DEFINES += -DAMREX_USE_SUNDIALS
+  INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include
+  LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR)
+
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_cvode
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_arkode
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nvecmanyvector
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_core
+
+  ifeq ($(USE_CUDA),TRUE)
+    LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nveccuda
+  endif
+
+endif
+
+include $(AMREX_HOME)/Src/Base/Make.package
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.rules
+

ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials

@@ -0,0 +1,64 @@
+SUNDIALS installation guide:
+https://computing.llnl.gov/projects/sundials/faq#inst
+
+Installation
+
+# You need SUNDIALS v7.1.1 or later.
+# Check https://computing.llnl.gov/projects/sundials/sundials-software to see if it's available for download.
+# If so, download sundials-x.y.z.tar.gz and extract it at the same level as amrex using
+>> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials
+>> mv sundials-x.y.z sundials-src
+
+# If v7.1.1. is not available on the website, clone the git repo directly and use the latest version
+# At the same level that amrex is cloned, do:
+
+>> git clone https://github.com/LLNL/sundials.git
+>> mv sundials sundials-src
+
+# Next
+
+>> mkdir sundials
+>> cd sundials
+
+######################
+HOST BUILD 
+######################
+
+>> mkdir instdir
+>> mkdir builddir
+>> cd builddir
+
+>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-src
+
+######################
+NVIDIA/CUDA BUILD
+######################
+
+# Navigate back to the 'sundials' directory and do:
+
+>> mkdir instdir_cuda
+>> mkdir builddir_cuda
+>> cd builddir_cuda
+
+>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-src
+
+######################
+
+>> make -j4
+>> make install
+
+# in your .bashrc or preferred configuration file, add the following (and then "source ~/.bashrc")
+
+# If you have a CPU build:
+
+>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir/lib/
+
+# If you have a NVIDIA/CUDA build:
+
+>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir_cuda/lib/
+
+# now you are ready to compile amrex-tutorials/ExampleCodes/SUNDIALS/Exec with:
+
+>> make -j4 # optional to have 'USE_CUDA=TRUE' as well
+
+# Refer to inputs to see how to enable different integrators

ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri

@@ -0,0 +1,70 @@
+##
+# Theoretical forward euler explicit time step constraint for each process is based 
+# on the diffusion and reaction coefficients.
+# For diffusion term
+# dt < dx^2 /(2*DIM*diffusion_coef) = (1/32)^2/(2*3*1) = 1.63e-4
+# For reaction term
+# dt <= 2/reaction_coef
+
+n_cell = 32
+max_grid_size = 16
+
+nsteps = 12 # 60 # 600
+plot_int = 2 # 10 # 100
+
+dt = 8.5e-3 # 1.7e-3 #1.7e-4
+
+# With integration.sundials.type = ERK, dt = 1.6e4 is stable, dt = 1.7e-4 is unstable
+# With integration.sundials.type = EX-MRI, dt = 1.7e-4 with fast_dt_ratio = 0.1 is stable
+# With integration.sundials.type = EX-MRI, dt = 1.7e-3 with fast_dt_ratio = 0.1 is stable
+# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.1 FAILS
+# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.02 is stable
+
+# To replicate heat equation
+diffusion_coef = 1.0
+reaction_coef = 1.e-4
+
+# MRI parameters
+use_MRI = true
+fast_dt_ratio = 0.02
+
+
+# Use adaptive time stepping and set integrator relative and absolute tolerances
+# adapt_dt = true
+# reltol   = 1.0e-4
+# abstol   = 1.0e-9
+
+# INTEGRATION
+# integration.type can take on the following values:
+# 0 or "ForwardEuler" => Native AMReX Forward Euler integrator
+# 1 or "RungeKutta"   => Native AMReX Explicit Runge Kutta controlled by integration.rk.type
+# 2 or "SUNDIALS"     => SUNDIALS backend controlled by integration.sundials.type
+#
+# If using the SUNDIALS Submodule, then compile with USE_SUNDIALS=TRUE or
+# AMReX_SUNDIALS=ON
+integration.type = SUNDIALS
+
+# Set the SUNDIALS method type:
+# ERK      = Explicit Runge-Kutta method
+# DIRK     = Diagonally Implicit Runge-Kutta method
+# IMEX-RK  = Implicit-Explicit Additive Runge-Kutta method
+# EX-MRI   = Explicit Multirate Infatesimal method
+# IM-MRI   = Implicit Multirate Infatesimal method
+# IMEX-MRI = Implicit-Explicit Multirate Infatesimal method
+#
+# Optionally select a specific SUNDIALS method by name, see the SUNDIALS
+# documentation for the supported method names
+integration.sundials.type = EX-MRI
+integration.sundials.fast_type = ERK
+
+## *** Select a specific SUNDIALS ERK method ***
+#integration.sundials.method = ARKODE_BOGACKI_SHAMPINE_4_2_3
+#
+## *** Select a specific SUNDIALS ImEx method ***
+#integration.sundials.method_i = ARKODE_ARK2_DIRK_3_1_2
+#integration.sundials.method_e = ARKODE_ARK2_ERK_3_1_2
+
+# *** Select a specific SUNDIALS MRI method ***
+integration.sundials.method = ARKODE_MIS_KW3
+integration.sundials.fast_method = ARKODE_KNOTH_WOLKE_3_3
+

ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package

@@ -0,0 +1,2 @@
+CEXE_sources += main.cpp
+CEXE_headers += myfunc.H