From c994b4f694ed05c260c9672ab3f4ff7d54c3198c Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 19 Feb 2020 16:43:03 -0500 Subject: [PATCH 1/5] split up the makefile to make it easier for other projects to include --- Make.CVODE | 25 ++++++++++ Make.Microphysics | 105 ++------------------------------------- Make.Microphysics_extern | 71 ++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 100 deletions(-) create mode 100644 Make.CVODE create mode 100644 Make.Microphysics_extern diff --git a/Make.CVODE b/Make.CVODE new file mode 100644 index 0000000000..1e6da2d33c --- /dev/null +++ b/Make.CVODE @@ -0,0 +1,25 @@ +ifeq ($(INTEGRATOR_DIR), CVODE) + + INCLUDE_LOCATIONS += $(CVODE_HOME)/include + + LIBRARY_LOCATIONS += $(CVODE_HOME)/lib + LIBRARIES += -lsundials_cvode -lsundials_sunlinsolband -lsundials_sunlinsolspbcgs -lsundials_sunlinsolsptfqmr -lsundials_sunmatrixsparse -lsundials_nveccuda -lsundials_sunlinsoldense -lsundials_sunlinsolspfgmr -lsundials_sunmatrixband -lsundials_sunnonlinsolfixedpoint -lsundials_nvecserial -lsundials_sunlinsolpcg -lsundials_sunlinsolspgmr -lsundials_sunmatrixdense -lsundials_sunnonlinsolnewton + +ifeq ($(USE_CUDA_CVODE),TRUE) + LIBRARIES += -rpath=$(CVODE_HOME)/lib +else + LIBRARIES += -Wl,-rpath,$(CVODE_HOME)/lib +endif + + ifeq ($(USE_CUDA_CVODE), TRUE) + DEFINES += -DCUDA_CVODE + endif + + ifeq ($(USE_CVODE_CUSOLVER), TRUE) + DEFINES += -DCVODE_CUSOLVER + + LIBRARIES += -lsundials_cusolver_sparse -lcusolver -lcusparse -lcublas + LIBRARIES += -rpath=$(COMPILE_CUDA_PATH)/lib64 + endif + +endif diff --git a/Make.Microphysics b/Make.Microphysics index 80ea999153..05ca7eb48b 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -97,37 +97,12 @@ endif #------------------------------------------------------------------------------ # CVODE directories #------------------------------------------------------------------------------ -ifeq ($(INTEGRATOR_DIR), CVODE) - - INCLUDE_LOCATIONS += $(CVODE_HOME)/include - - LIBRARY_LOCATIONS += $(CVODE_HOME)/lib - LIBRARIES += -lsundials_cvode -lsundials_sunlinsolband -lsundials_sunlinsolspbcgs -lsundials_sunlinsolsptfqmr -lsundials_sunmatrixsparse -lsundials_nveccuda -lsundials_sunlinsoldense -lsundials_sunlinsolspfgmr -lsundials_sunmatrixband -lsundials_sunnonlinsolfixedpoint -lsundials_nvecserial -lsundials_sunlinsolpcg -lsundials_sunlinsolspgmr -lsundials_sunmatrixdense -lsundials_sunnonlinsolnewton - -ifeq ($(USE_CUDA_CVODE),TRUE) - LIBRARIES += -rpath=$(CVODE_HOME)/lib -else - LIBRARIES += -Wl,-rpath,$(CVODE_HOME)/lib -endif - - ifeq ($(USE_CUDA_CVODE), TRUE) - DEFINES += -DCUDA_CVODE - endif - - ifeq ($(USE_CVODE_CUSOLVER), TRUE) - DEFINES += -DCVODE_CUSOLVER - - LIBRARIES += -lsundials_cusolver_sparse -lcusolver -lcusparse -lcublas - LIBRARIES += -rpath=$(COMPILE_CUDA_PATH)/lib64 - endif - -endif +include $(MICROPHYSICS_HOME)/Make.CVODE #------------------------------------------------------------------------------ # Microphysics directories #------------------------------------------------------------------------------ - -Bdirs := interfaces constants +Bdirs := constants Bpack += $(foreach dir, $(Bdirs), $(TOP)/$(dir)/Make.package) Blocs += $(foreach dir, $(Bdirs), $(TOP)/$(dir)) @@ -147,88 +122,18 @@ else Pdirs := Base endif -Bpack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) +Bpack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) #------------------------------------------------------------------------------ -# microphysics +# microphysics #------------------------------------------------------------------------------ - -EOS_PATH := $(MICROPHYSICS_HOME)/EOS/$(strip $(EOS_DIR)) -NETWORK_PATH := $(MICROPHYSICS_HOME)/networks/$(strip $(NETWORK_DIR)) -CONDUCTIVITY_PATH := $(MICROPHYSICS_HOME)/conductivity/$(strip $(CONDUCTIVITY_DIR)) - - -# Problems can specify the input file for general_null by setting -# GENERAL_NET_INPUTS directly (full path) or just specifying the name -# of the inputs file as NETWORK_INPUTS and let us sort it out -ifndef GENERAL_NET_INPUTS - ifneq "$(wildcard $(NETWORK_INPUTS))" "" - GENERAL_NET_INPUTS := $(NETWORK_INPUTS) - else - GENERAL_NET_INPUTS := $(MICROPHYSICS_HOME)/networks/general_null/$(NETWORK_INPUTS) - endif -endif - -# URCA network has tables -ifeq ($(findstring URCA-simple, $(NETWORK_DIR)), URCA-simple) - all: urcatables -endif - -urcatables: - @if [ ! -f 23Ne-23Na_betadecay.dat ]; then echo ${bold}Linking 23Ne-23Na_betadecay.dat${normal}; ln -s $(NETWORK_PATH)/23Ne-23Na_betadecay.dat .; fi - @if [ ! -f 23Na-23Ne_electroncapture.dat ]; then echo ${bold}Linking 23Na-23Ne_electroncapture.dat${normal}; ln -s $(NETWORK_PATH)/23Na-23Ne_electroncapture.dat .; fi - -# Pynucastro networks read rate data from an external file -PYNUCASTRO_FILE := "$(NETWORK_PATH)/reaclib_rate_metadata.dat" -all: pynucastrorates - - -# do not check to see if the file already exists, just force-link it in -# because different Reaclib networks will have different versions of the file -pynucastrorates: - @if [ -f $(PYNUCASTRO_FILE) ] && [ ! -f ./reaclib_rate_metadata.dat ]; then echo Linking pynucastro rate file; ln -s $(PYNUCASTRO_FILE) .; fi - -EXTERN_CORE += $(EOS_HOME) -EXTERN_CORE += $(EOS_PATH) - -# the helmholtz EOS has an include file -- also add a target to link -# the table into the problem directory. -ifeq ($(findstring helmholtz, $(EOS_DIR)), helmholtz) - all: table -endif - -table: - @if [ ! -f helm_table.dat ]; then echo Linking helm_table.dat; ln -s $(EOS_PATH)/helm_table.dat .; fi - -ifeq ($(findstring gamma_law_general, $(EOS_DIR)), gamma_law_general) - DEFINES += -DEOS_GAMMA_LAW_GENERAL -endif - -# include the conductivity -ifdef USE_CONDUCTIVITY - ifeq ($(USE_CONDUCTIVITY),TRUE) - DEFINES += -DCONDUCTIVITY - EXTERN_CORE += $(CONDUCTIVITY_HOME) - EXTERN_CORE += $(CONDUCTIVITY_PATH) - endif -endif - -# include the network -EXTERN_CORE += $(NETWORK_PATH) - -# Note that this should be located after the include of the EOS and -# Network includes since it has actions that depend on variables set -# there. - -EXTERN_CORE += $(MICROPHYSICS_HOME)/EOS -EXTERN_CORE += $(MICROPHYSICS_HOME)/networks +include $(MICROPHYSICS_HOME)/Make.Microphysics_extern Bpack += $(foreach dir, $(EXTERN_CORE), $(dir)/Make.package) Blocs += $(foreach dir, $(EXTERN_CORE), $(dir)) - #------------------------------------------------------------------------------ # include all of the necessary directories #------------------------------------------------------------------------------ diff --git a/Make.Microphysics_extern b/Make.Microphysics_extern new file mode 100644 index 0000000000..21ab9612a3 --- /dev/null +++ b/Make.Microphysics_extern @@ -0,0 +1,71 @@ + +EOS_PATH := $(MICROPHYSICS_HOME)/EOS/$(strip $(EOS_DIR)) +NETWORK_PATH := $(MICROPHYSICS_HOME)/networks/$(strip $(NETWORK_DIR)) +CONDUCTIVITY_PATH := $(MICROPHYSICS_HOME)/conductivity/$(strip $(CONDUCTIVITY_DIR)) + + +# Problems can specify the input file for general_null by setting +# GENERAL_NET_INPUTS directly (full path) or just specifying the name +# of the inputs file as NETWORK_INPUTS and let us sort it out +ifndef GENERAL_NET_INPUTS + ifneq "$(wildcard $(NETWORK_INPUTS))" "" + GENERAL_NET_INPUTS := $(NETWORK_INPUTS) + else + GENERAL_NET_INPUTS := $(MICROPHYSICS_HOME)/networks/general_null/$(NETWORK_INPUTS) + endif +endif + +# URCA network has tables +ifeq ($(findstring URCA-simple, $(NETWORK_DIR)), URCA-simple) + all: urcatables +endif + +urcatables: + @if [ ! -f 23Ne-23Na_betadecay.dat ]; then echo ${bold}Linking 23Ne-23Na_betadecay.dat${normal}; ln -s $(NETWORK_PATH)/23Ne-23Na_betadecay.dat .; fi + @if [ ! -f 23Na-23Ne_electroncapture.dat ]; then echo ${bold}Linking 23Na-23Ne_electroncapture.dat${normal}; ln -s $(NETWORK_PATH)/23Na-23Ne_electroncapture.dat .; fi + +# Pynucastro networks read rate data from an external file +PYNUCASTRO_FILE := "$(NETWORK_PATH)/reaclib_rate_metadata.dat" +all: pynucastrorates + + +# do not check to see if the file already exists, just force-link it in +# because different Reaclib networks will have different versions of the file +pynucastrorates: + @if [ -f $(PYNUCASTRO_FILE) ] && [ ! -f ./reaclib_rate_metadata.dat ]; then echo Linking pynucastro rate file; ln -s $(PYNUCASTRO_FILE) .; fi + +EXTERN_CORE += $(EOS_HOME) +EXTERN_CORE += $(EOS_PATH) + +# the helmholtz EOS has an include file -- also add a target to link +# the table into the problem directory. +ifeq ($(findstring helmholtz, $(EOS_DIR)), helmholtz) + all: table +endif + +table: + @if [ ! -f helm_table.dat ]; then echo Linking helm_table.dat; ln -s $(EOS_PATH)/helm_table.dat .; fi + +ifeq ($(findstring gamma_law_general, $(EOS_DIR)), gamma_law_general) + DEFINES += -DEOS_GAMMA_LAW_GENERAL +endif + +# include the conductivity +ifdef USE_CONDUCTIVITY + ifeq ($(USE_CONDUCTIVITY),TRUE) + DEFINES += -DCONDUCTIVITY + EXTERN_CORE += $(CONDUCTIVITY_HOME) + EXTERN_CORE += $(CONDUCTIVITY_PATH) + endif +endif + +# include the network +EXTERN_CORE += $(NETWORK_PATH) + +# Note that this should be located after the include of the EOS and +# Network includes since it has actions that depend on variables set +# there. + +EXTERN_CORE += $(MICROPHYSICS_HOME)/EOS +EXTERN_CORE += $(MICROPHYSICS_HOME)/networks +EXTERN_CORE += $(MICROPHYSICS_HOME)/interfaces From 99fc88a4db03fa0f09eb1779b32b0522160b91f8 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 19 Feb 2020 19:30:01 -0500 Subject: [PATCH 2/5] some more Make reorgs --- Make.Microphysics | 6 ++--- Make.Microphysics_extern | 38 +++++++++++++++++++++--------- integration/Make.package | 7 ++++++ networks/general_null/Make.package | 2 -- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Make.Microphysics b/Make.Microphysics index 05ca7eb48b..e6f4565725 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -67,10 +67,6 @@ ifeq ($(USE_ACC), TRUE) DEFINES += -DACC endif -ifeq ($(USE_EXTRA_THERMO), TRUE) - DEFINES += -DEXTRA_THERMO -endif - ifeq ($(USE_SIMPLIFIED_SDC), TRUE) DEFINES += -DSIMPLIFIED_SDC endif @@ -128,6 +124,8 @@ Bpack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) #------------------------------------------------------------------------------ # microphysics #------------------------------------------------------------------------------ +NETWORK_OUTPUT_PATH := $(CURDIR) + include $(MICROPHYSICS_HOME)/Make.Microphysics_extern Bpack += $(foreach dir, $(EXTERN_CORE), $(dir)/Make.package) diff --git a/Make.Microphysics_extern b/Make.Microphysics_extern index 21ab9612a3..ea8f6365b6 100644 --- a/Make.Microphysics_extern +++ b/Make.Microphysics_extern @@ -1,8 +1,9 @@ EOS_PATH := $(MICROPHYSICS_HOME)/EOS/$(strip $(EOS_DIR)) NETWORK_PATH := $(MICROPHYSICS_HOME)/networks/$(strip $(NETWORK_DIR)) -CONDUCTIVITY_PATH := $(MICROPHYSICS_HOME)/conductivity/$(strip $(CONDUCTIVITY_DIR)) - +ifeq ($(USE_CONDUCTIVITY), TRUE) + CONDUCTIVITY_PATH := $(MICROPHYSICS_HOME)/conductivity/$(strip $(CONDUCTIVITY_DIR)) +endif # Problems can specify the input file for general_null by setting # GENERAL_NET_INPUTS directly (full path) or just specifying the name @@ -50,18 +51,10 @@ ifeq ($(findstring gamma_law_general, $(EOS_DIR)), gamma_law_general) DEFINES += -DEOS_GAMMA_LAW_GENERAL endif -# include the conductivity -ifdef USE_CONDUCTIVITY - ifeq ($(USE_CONDUCTIVITY),TRUE) - DEFINES += -DCONDUCTIVITY - EXTERN_CORE += $(CONDUCTIVITY_HOME) - EXTERN_CORE += $(CONDUCTIVITY_PATH) - endif -endif - # include the network EXTERN_CORE += $(NETWORK_PATH) + # Note that this should be located after the include of the EOS and # Network includes since it has actions that depend on variables set # there. @@ -69,3 +62,26 @@ EXTERN_CORE += $(NETWORK_PATH) EXTERN_CORE += $(MICROPHYSICS_HOME)/EOS EXTERN_CORE += $(MICROPHYSICS_HOME)/networks EXTERN_CORE += $(MICROPHYSICS_HOME)/interfaces + +# include the conductivity +ifeq ($(USE_CONDUCTIVITY),TRUE) + DEFINES += -DCONDUCTIVITY + EXTERN_CORE += $(CONDUCTIVITY_HOME) + EXTERN_CORE += $(CONDUCTIVITY_PATH) +endif + + +ifeq ($(USE_RAD), TRUE) + ifeq ($(Opacity_dir), null) + $(error The null opacity directory has been removed, please update to rad_power_law) + endif + OPAC_PATH := $(MICROPHYSICS_HOME)/opacity/$(Opacity_dir) + EXTERN_CORE += $(OPAC_PATH) +endif + +ifeq ($(USE_EXTRA_THERMO), TRUE) + DEFINES += -DEXTRA_THERMO +endif + + + diff --git a/integration/Make.package b/integration/Make.package index c8ee77d830..154f574e33 100644 --- a/integration/Make.package +++ b/integration/Make.package @@ -19,6 +19,13 @@ endif DEFINES += -DINTEGRATOR=$(INTEGRATOR_NUM) +ifeq ($(USE_CUDA), TRUE) + ifneq ($(INTEGRATOR_DIR), VODE90) + $(error Only the VODE90 integrator supports GPUs currently, please change INTEGRATOR_DIR) + endif +endif + + # Include all integrators in the build. ifeq ($(INTEGRATOR_DIR),VODE) diff --git a/networks/general_null/Make.package b/networks/general_null/Make.package index 156b5f0613..a1f06af810 100644 --- a/networks/general_null/Make.package +++ b/networks/general_null/Make.package @@ -8,8 +8,6 @@ endif f90EXE_sources += actual_rhs.f90 endif -NETWORK_OUTPUT_PATH ?= $(CURDIR) - $(NETWORK_OUTPUT_PATH)/network_properties.H: $(NETWORK_OUTPUT_PATH)/actual_network.F90 # actual_network.F90 is created at build time for this network From 87edfe35ada76da90a568ed32ef482d2fef7a1ee Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 19 Feb 2020 20:04:23 -0500 Subject: [PATCH 3/5] finish a reorg of the makefiles -- this will let other applications more easily include them --- Make.Microphysics | 2 +- Make.Microphysics_extern | 2 ++ networks/Make.package | 51 ++++++++++++++++++---------------------- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Make.Microphysics b/Make.Microphysics index e6f4565725..96c1917e51 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -124,7 +124,7 @@ Bpack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) #------------------------------------------------------------------------------ # microphysics #------------------------------------------------------------------------------ -NETWORK_OUTPUT_PATH := $(CURDIR) +NETWORK_OUTPUT_PATH := . include $(MICROPHYSICS_HOME)/Make.Microphysics_extern diff --git a/Make.Microphysics_extern b/Make.Microphysics_extern index ea8f6365b6..fbdbe386da 100644 --- a/Make.Microphysics_extern +++ b/Make.Microphysics_extern @@ -1,3 +1,5 @@ +# This is the main include makefile for applications that want to use Microphysics +# You should set NETWORK_OUTPUT_PATH before including this file EOS_PATH := $(MICROPHYSICS_HOME)/EOS/$(strip $(EOS_DIR)) NETWORK_PATH := $(MICROPHYSICS_HOME)/networks/$(strip $(NETWORK_DIR)) diff --git a/networks/Make.package b/networks/Make.package index 32effc4207..4de27a2a01 100644 --- a/networks/Make.package +++ b/networks/Make.package @@ -7,38 +7,35 @@ include $(MICROPHYSICS_UTIL_PATH)/Make.package ifeq ($(USE_REACT), TRUE) -ifeq ($(INTEGRATOR_DIR), VODE90) - override INTEGRATOR_DIR = VODE - $(warn VODE90 has been renamed VODE) -endif + ifeq ($(INTEGRATOR_DIR), VODE90) + override INTEGRATOR_DIR = VODE + $(warn VODE90 has been renamed VODE) + endif -ifneq ($(INTEGRATOR_DIR), VODE) + ifneq ($(INTEGRATOR_DIR), VODE) - INCLUDE_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/LINPACK - VPATH_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/LINPACK + INCLUDE_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/LINPACK + VPATH_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/LINPACK - include $(MICROPHYSICS_UTIL_PATH)/LINPACK/Make.package -else - ifeq ($(USE_CLEANED_CORRECTOR), TRUE) - DEFINES += -DCLEAN_INTEGRATOR_CORRECTION + include $(MICROPHYSICS_UTIL_PATH)/LINPACK/Make.package + else + ifeq ($(USE_CLEANED_CORRECTOR), TRUE) + DEFINES += -DCLEAN_INTEGRATOR_CORRECTION + endif endif -endif -ifneq ($(USE_SYSTEM_BLAS), TRUE) - ifneq ($(INTEGRATOR_DIR), VODE) - INCLUDE_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/BLAS - VPATH_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/BLAS + ifneq ($(USE_SYSTEM_BLAS), TRUE) + ifneq ($(INTEGRATOR_DIR), VODE) + INCLUDE_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/BLAS + VPATH_LOCATIONS += $(MICROPHYSICS_UTIL_PATH)/BLAS - include $(MICROPHYSICS_UTIL_PATH)/BLAS/Make.package + include $(MICROPHYSICS_UTIL_PATH)/BLAS/Make.package + endif endif -endif -VPATH_LOCATIONS += $(MICROPHYSICS_HOME)/networks -f90EXE_sources += rate_type.f90 + VPATH_LOCATIONS += $(MICROPHYSICS_HOME)/networks + f90EXE_sources += rate_type.f90 -endif - -ifeq ($(USE_REACT), TRUE) # we need the actual integrator in the VPATH before the # integration/ dir to get overrides correct @@ -88,15 +85,13 @@ ifeq ($(USE_NEUTRINOS), TRUE) endif -NETWORK_HEADERS_DIR ?= . - ifneq ($(NETWORK_DIR), general_null) -$(NETWORK_HEADERS_DIR)/network_properties.H: $(NETWORK_HEADERS_DIR)/network_properties.F90 +$(NETWORK_OUTPUT_PATH)/network_properties.H: $(NETWORK_OUTPUT_PATH)/network_properties.F90 -$(NETWORK_HEADERS_DIR)/network_properties.F90: +$(NETWORK_OUTPUT_PATH)/network_properties.F90: $(MICROPHYSICS_HOME)/networks/update_headers.py \ --microphysics_path $(MICROPHYSICS_HOME) \ --net $(NETWORK_DIR) \ - --odir $(NETWORK_HEADERS_DIR) + --odir $(NETWORK_OUTPUT_PATH) endif From c6b93e0d1a8b4e20fa602a1118a856aa579f4605 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 20 Feb 2020 10:49:30 -0500 Subject: [PATCH 4/5] VODE90 -> VODE --- integration/Make.package | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/Make.package b/integration/Make.package index 154f574e33..9d0057e2a1 100644 --- a/integration/Make.package +++ b/integration/Make.package @@ -20,8 +20,8 @@ endif DEFINES += -DINTEGRATOR=$(INTEGRATOR_NUM) ifeq ($(USE_CUDA), TRUE) - ifneq ($(INTEGRATOR_DIR), VODE90) - $(error Only the VODE90 integrator supports GPUs currently, please change INTEGRATOR_DIR) + ifneq ($(INTEGRATOR_DIR), VODE) + $(error Only the VODE integrator supports GPUs currently, please change INTEGRATOR_DIR) endif endif From f20d2a03024423a392fa2a90734455269d1abd4f Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 20 Feb 2020 11:09:58 -0500 Subject: [PATCH 5/5] set a default for NETWORK_OUTPUT_PATH --- networks/Make.package | 2 ++ 1 file changed, 2 insertions(+) diff --git a/networks/Make.package b/networks/Make.package index 4de27a2a01..3daf1b1c53 100644 --- a/networks/Make.package +++ b/networks/Make.package @@ -85,6 +85,8 @@ ifeq ($(USE_NEUTRINOS), TRUE) endif +NETWORK_OUTPUT_PATH ?= . + ifneq ($(NETWORK_DIR), general_null) $(NETWORK_OUTPUT_PATH)/network_properties.H: $(NETWORK_OUTPUT_PATH)/network_properties.F90