Skip to content

Commit

Permalink
Merge branch 'main' into sundials_work
Browse files Browse the repository at this point in the history
  • Loading branch information
ajnonaka committed Sep 5, 2024
2 parents 072d5d0 + b5520c2 commit c5a3a7e
Show file tree
Hide file tree
Showing 12 changed files with 511 additions and 10 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/dependencies/dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@ sudo apt-get install -y --no-install-recommends\
python3 \
python3-pip

python3 -m pip install -U pip setuptools wheel
python3 -m pip install -U pip
python3 -m pip install -U build packaging setuptools wheel
3 changes: 2 additions & 1 deletion .github/workflows/dependencies/dependencies_clang6.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@ sudo apt-get install -y \
python3 \
python3-pip

python3 -m pip install -U pip setuptools wheel
python3 -m pip install -U pip
python3 -m pip install -U build packaging setuptools wheel
3 changes: 2 additions & 1 deletion .github/workflows/dependencies/dependencies_gcc10.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ sudo apt-get install -y --no-install-recommends \
python3 \
python3-pip

python3 -m pip install -U pip setuptools wheel
python3 -m pip install -U pip
python3 -m pip install -U build packaging setuptools wheel
3 changes: 2 additions & 1 deletion .github/workflows/dependencies/dependencies_nofortran.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ sudo apt-get install -y --no-install-recommends\
python3 \
python3-pip

python3 -m pip install -U pip setuptools wheel
python3 -m pip install -U pip
python3 -m pip install -U build packaging setuptools wheel
15 changes: 15 additions & 0 deletions ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
if (AMReX_SPACEDIM EQUAL 1)
return()
endif ()

# List of source files
set(_sources main.cpp myfunc.H)
list(TRANSFORM _sources PREPEND "Source/")

# List of input files
file( GLOB_RECURSE _input_files LIST_DIRECTORIES false Exec/input* )

setup_tutorial(_sources _input_files)

unset( _sources )
unset( _input_files )
51 changes: 51 additions & 0 deletions ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# AMREX_HOME defines the directory in which we will find all the AMReX code.
AMREX_HOME ?= ../../../../../amrex

DEBUG = FALSE
USE_MPI = TRUE
USE_OMP = FALSE
COMP = gnu
DIM = 3
USE_RPATH = TRUE
USE_SUNDIALS = TRUE

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

include ../Source/Make.package
VPATH_LOCATIONS += ../Source
INCLUDE_LOCATIONS += ../Source

ifeq ($(USE_SUNDIALS),TRUE)
ifeq ($(USE_CUDA),TRUE)
SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir_cuda
else
SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir
endif
ifeq ($(NERSC_HOST),perlmutter)
SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib64
else
SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib
endif

USE_CVODE_LIBS ?= TRUE
USE_ARKODE_LIBS ?= TRUE

DEFINES += -DAMREX_USE_SUNDIALS
INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include
LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR)

LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_cvode
LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_arkode
LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nvecmanyvector
LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_core

ifeq ($(USE_CUDA),TRUE)
LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nveccuda
endif

endif

include $(AMREX_HOME)/Src/Base/Make.package

include $(AMREX_HOME)/Tools/GNUMake/Make.rules

64 changes: 64 additions & 0 deletions ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
SUNDIALS installation guide:
https://computing.llnl.gov/projects/sundials/faq#inst

Installation

# You need SUNDIALS v7.1.1 or later.
# Check https://computing.llnl.gov/projects/sundials/sundials-software to see if it's available for download.
# If so, download sundials-x.y.z.tar.gz and extract it at the same level as amrex using
>> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials
>> mv sundials-x.y.z sundials-src

# If v7.1.1. is not available on the website, clone the git repo directly and use the latest version
# At the same level that amrex is cloned, do:

>> git clone https://github.com/LLNL/sundials.git
>> mv sundials sundials-src

# Next

>> mkdir sundials
>> cd sundials

######################
HOST BUILD
######################

>> mkdir instdir
>> mkdir builddir
>> cd builddir

>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-src

######################
NVIDIA/CUDA BUILD
######################

# Navigate back to the 'sundials' directory and do:

>> mkdir instdir_cuda
>> mkdir builddir_cuda
>> cd builddir_cuda

>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-src

######################

>> make -j4
>> make install

# in your .bashrc or preferred configuration file, add the following (and then "source ~/.bashrc")

# If you have a CPU build:

>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir/lib/

# If you have a NVIDIA/CUDA build:

>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir_cuda/lib/

# now you are ready to compile amrex-tutorials/ExampleCodes/SUNDIALS/Exec with:

>> make -j4 # optional to have 'USE_CUDA=TRUE' as well

# Refer to inputs to see how to enable different integrators
70 changes: 70 additions & 0 deletions ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
##
# Theoretical forward euler explicit time step constraint for each process is based
# on the diffusion and reaction coefficients.
# For diffusion term
# dt < dx^2 /(2*DIM*diffusion_coef) = (1/32)^2/(2*3*1) = 1.63e-4
# For reaction term
# dt <= 2/reaction_coef

n_cell = 32
max_grid_size = 16

nsteps = 12 # 60 # 600
plot_int = 2 # 10 # 100

dt = 8.5e-3 # 1.7e-3 #1.7e-4

# With integration.sundials.type = ERK, dt = 1.6e4 is stable, dt = 1.7e-4 is unstable
# With integration.sundials.type = EX-MRI, dt = 1.7e-4 with fast_dt_ratio = 0.1 is stable
# With integration.sundials.type = EX-MRI, dt = 1.7e-3 with fast_dt_ratio = 0.1 is stable
# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.1 FAILS
# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.02 is stable

# To replicate heat equation
diffusion_coef = 1.0
reaction_coef = 1.e-4

# MRI parameters
use_MRI = true
fast_dt_ratio = 0.02


# Use adaptive time stepping and set integrator relative and absolute tolerances
# adapt_dt = true
# reltol = 1.0e-4
# abstol = 1.0e-9

# INTEGRATION
# integration.type can take on the following values:
# 0 or "ForwardEuler" => Native AMReX Forward Euler integrator
# 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type
# 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.type
#
# If using the SUNDIALS Submodule, then compile with USE_SUNDIALS=TRUE or
# AMReX_SUNDIALS=ON
integration.type = SUNDIALS

# Set the SUNDIALS method type:
# ERK = Explicit Runge-Kutta method
# DIRK = Diagonally Implicit Runge-Kutta method
# IMEX-RK = Implicit-Explicit Additive Runge-Kutta method
# EX-MRI = Explicit Multirate Infatesimal method
# IM-MRI = Implicit Multirate Infatesimal method
# IMEX-MRI = Implicit-Explicit Multirate Infatesimal method
#
# Optionally select a specific SUNDIALS method by name, see the SUNDIALS
# documentation for the supported method names
integration.sundials.type = EX-MRI
integration.sundials.fast_type = ERK

## *** Select a specific SUNDIALS ERK method ***
#integration.sundials.method = ARKODE_BOGACKI_SHAMPINE_4_2_3
#
## *** Select a specific SUNDIALS ImEx method ***
#integration.sundials.method_i = ARKODE_ARK2_DIRK_3_1_2
#integration.sundials.method_e = ARKODE_ARK2_ERK_3_1_2

# *** Select a specific SUNDIALS MRI method ***
integration.sundials.method = ARKODE_MIS_KW3
integration.sundials.fast_method = ARKODE_KNOTH_WOLKE_3_3

2 changes: 2 additions & 0 deletions ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
CEXE_sources += main.cpp
CEXE_headers += myfunc.H
Loading

0 comments on commit c5a3a7e

Please sign in to comment.