From 58a368faf61b712337173dbb360bfe53e5c480d1 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 19 Aug 2019 10:47:25 -0400 Subject: [PATCH 01/72] implement MOL/SDC PLM reflecting BCs --- .../science/flame_wave/inputs.boost_10_10.sdc | 2 +- Source/hydro/Castro_hydro_F.H | 2 +- Source/hydro/Castro_mol_hydro.cpp | 2 +- Source/hydro/Castro_mol_nd.F90 | 121 +++++++++++++++++- 4 files changed, 118 insertions(+), 9 deletions(-) diff --git a/Exec/science/flame_wave/inputs.boost_10_10.sdc b/Exec/science/flame_wave/inputs.boost_10_10.sdc index f787ea646c..fb880d2960 100644 --- a/Exec/science/flame_wave/inputs.boost_10_10.sdc +++ b/Exec/science/flame_wave/inputs.boost_10_10.sdc @@ -14,7 +14,7 @@ amr.n_cell = 768 192 # 1 = Inflow 4 = SlipWall # 2 = Outflow 5 = NoSlipWall # >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< -castro.lo_bc = 3 1 +castro.lo_bc = 3 3 castro.hi_bc = 2 1 castro.yl_ext_bc_type = "hse" diff --git a/Source/hydro/Castro_hydro_F.H b/Source/hydro/Castro_hydro_F.H index 5d78e30e52..4ff6d6f703 100644 --- a/Source/hydro/Castro_hydro_F.H +++ b/Source/hydro/Castro_hydro_F.H @@ -414,7 +414,7 @@ extern "C" BL_FORT_FAB_ARG_3D(dq), BL_FORT_FAB_ARG_3D(qm), BL_FORT_FAB_ARG_3D(qp), - const amrex::Real* dx); + const int* domlo, const int* domhi); void ca_mol_ppm_reconstruct (const int* lo, const int* hi, diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index 1ffb918480..441e39c3a1 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -224,7 +224,7 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) BL_TO_FORTRAN_ANYD(dq), BL_TO_FORTRAN_ANYD(qm), BL_TO_FORTRAN_ANYD(qp), - AMREX_REAL_ANYD(dx)); + AMREX_INT_ANYD(domain_lo), AMREX_INT_ANYD(domain_hi)); } else { diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index db9c13eeaf..4a8afe5b0d 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -13,13 +13,13 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & dq, dq_lo, dq_hi, & qm, qm_lo, qm_hi, & qp, qp_lo, qp_hi, & - dx) bind(C, name="ca_mol_plm_reconstruct") + domlo, domhi) bind(C, name="ca_mol_plm_reconstruct") use castro_error_module use meth_params_module, only : NQ, NVAR, NGDNV, GDPRES, & UTEMP, UMX, & use_flattening, QPRES, & - QTEMP, QFS, QFX, QREINT, QRHO, & + QTEMP, QFS, QFX, QREINT, QRHO, QU, QV, QW, & first_order_hydro, hybrid_riemann, & ppm_temp_fix use amrex_constants_module, only : ZERO, HALF, ONE, FOURTH @@ -28,7 +28,7 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & use eos_type_module, only : eos_t, eos_input_rt use eos_module, only : eos use network, only : nspec, naux - use prob_params_module, only : dg, coord_type + use prob_params_module, only : dg, coord_type, Symmetry, physbc_lo, physbc_hi implicit none @@ -44,7 +44,7 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & real(rt), intent(inout) :: dq(dq_lo(1):dq_hi(1), dq_lo(2):dq_hi(2), dq_lo(3):dq_hi(3), NQ) real(rt), intent(inout) :: qm(qm_lo(1):qm_hi(1), qm_lo(2):qm_hi(2), qm_lo(3):qm_hi(3), NQ, AMREX_SPACEDIM) real(rt), intent(inout) :: qp(qp_lo(1):qp_hi(1), qp_lo(2):qp_hi(2), qp_lo(3):qp_hi(3), NQ, AMREX_SPACEDIM) - real(rt), intent(in) :: dx(3) + integer, intent(in) :: domlo(3), domhi(3) integer :: idir, i, j, k, n type (eos_t) :: eos_state @@ -117,8 +117,117 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & #endif end if - end do - end do + + end do ! component loop + + ! special care for reflecting BCs + if (idir == 1) then + if (lo(1) == domlo(1)) then + + do k = lo(3)-dg(3), hi(3)+dg(3) + do j = lo(2)-dg(2), hi(2)+dg(2) + + ! reset the left state at domlo(1) if needed -- it is outside the domain + if (physbc_lo(1) == Symmetry) then + qm(domlo(1),j,k,:,1) = qp(domlo(1),j,k,:,1) + qm(domlo(1),j,k,QU,1) = -qp(domlo(1),j,k,QU,1) + end if + + end do + end do + + end if + + if (hi(1)+1 == domhi(1)+1) then + ! reset the right state at domhi(1)+1 if needed -- it is outside the domain + + do k = lo(3)-dg(3), hi(3)+dg(3) + do j = lo(2)-dg(2), hi(2)+dg(2) + + if (physbc_hi(1) == Symmetry) then + qp(domhi(1)+1,j,k,:,1) = qm(domhi(1)+1,j,k,:,1) + qp(domhi(1)+1,j,k,QU,1) = -qm(domhi(1)+1,j,k,QU,1) + end if + + end do + end do + + end if + +#if AMREX_SPACEDIM >= 2 + else if (idir == 2) then + + if (lo(2) == domlo(2)) then + ! reset the left state at domlo(2) if needed -- it is outside the domain + + do k = lo(3)-dg(3), hi(3)+dg(3) + do i = lo(1)-1, hi(1)+1 + + if (physbc_lo(2) == Symmetry) then + qm(i,domlo(2),k,:,2) = qp(i,domlo(2),k,:,2) + qm(i,domlo(2),k,QV,2) = -qp(i,domlo(2),k,QV,2) + end if + + end do + end do + + end if + + if (hi(2)+1 == domhi(2)+1) then + ! reset the right state at domhi(2)+1 if needed -- it is outside the domain + + do k = lo(3)-dg(3), hi(3)+dg(3) + do i = lo(1)-1, hi(1)+1 + + if (physbc_hi(2) == Symmetry) then + qp(i,domhi(2)+1,k,:,2) = qm(i,domhi(2)+1,k,:,2) + qp(i,domhi(2)+1,k,QV,2) = -qm(i,domhi(2)+1,k,QV,2) + end if + + end do + end do + + end if +#endif +#if AMREX_SPACEDIM == 3 + else + + if (lo(3) == domlo(3)) then + ! reset the left state at domlo(3) if needed -- it is outside the domain + + do j = lo(2)-1, hi(2)+1 + do i = lo(1)-1, hi(1)+1 + + if (physbc_lo(3) == Symmetry) then + qm(i,j,domlo(3),:,3) = qp(i,j,domlo(3),:,3) + qm(i,j,domlo(3),QW,3) = -qp(i,j,domlo(3),QW,3) + end if + + end do + end do + + end if + + if (hi(3)+1 == domhi(3)+1) then + ! reset the right state at domhi(3)+1 if needed -- it is outside the domain + + do j = lo(2)-1, hi(2)+1 + do i = lo(1)-1, hi(1)+1 + + if (physbc_hi(3) == Symmetry) then + qp(i,j,domhi(3)+1,:,3) = qm(i,j,domhi(3)+1,:,3) + qp(i,j,domhi(3)+1,QW,3) = -qm(i,j,domhi(3)+1,QW,3) + end if + + end do + end do + + end if +#endif + + end if + + end do ! dimension loop ! use T to define p if (ppm_temp_fix == 1) then From 79d7a44a8f919481d306eed28f7e5565c534bcab Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 19 Aug 2019 13:00:10 -0400 Subject: [PATCH 02/72] implement an okay_to_burn() function --- Source/driver/sdc_util.F90 | 41 ++++++++++++++++++++------------- Source/reactions/React_nd.F90 | 6 ++--- Source/reactions/react_util.F90 | 21 +++++++++++++++++ 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index d9348f8045..26df99a566 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1076,7 +1076,7 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & ! update k_m to k_n via advection -- this is a second-order accurate update use meth_params_module, only : NVAR - use amrex_constants_module, only : HALF + use amrex_constants_module, only : ZERO, HALF use burn_type_module, only : burn_t use network, only : nspec, nspec_evolve use react_util_module @@ -1126,23 +1126,32 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & HALF * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + & HALF * (R_0_old(i,j,k,:) + R_1_old(i,j,k,:)) - ! this is the full state -- this will be updated as we - ! solve the nonlinear system. We want to start with a - ! good initial guess. For later iterations, we should - ! begin with the result from the previous iteration. For - ! the first iteration, let's try to extrapolate forward - ! in time. - if (sdc_iteration == 0) then - U_new(:) = U_old(:) + dt_m * A_m(i,j,k,:) + dt_m * R_0_old(i,j,k,:) - else - U_new(:) = k_n(i,j,k,:) - endif + ! only burn if we are within the temperature and density + ! limits for burning + if (.not. okay_to_burn(U_old)) then + R_full(:) = ZERO - call sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) + else - ! we solved our system to some tolerance, but let's be sure we are conservative by - ! reevaluating the reactions and then doing the full step update - call single_zone_react_source(U_new, R_full, i, j, k, burn_state) + ! this is the full state -- this will be updated as we + ! solve the nonlinear system. We want to start with a + ! good initial guess. For later iterations, we should + ! begin with the result from the previous iteration. For + ! the first iteration, let's try to extrapolate forward + ! in time. + if (sdc_iteration == 0) then + U_new(:) = U_old(:) + dt_m * A_m(i,j,k,:) + dt_m * R_0_old(i,j,k,:) + else + U_new(:) = k_n(i,j,k,:) + endif + + call sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) + + ! we solved our system to some tolerance, but let's be sure we are conservative by + ! reevaluating the reactions and then doing the full step update + call single_zone_react_source(U_new, R_full, i, j, k, burn_state) + + end if U_new(:) = U_old(:) + dt_m * R_full(:) + dt_m * C(:) diff --git a/Source/reactions/React_nd.F90 b/Source/reactions/React_nd.F90 index b74e769d3b..4f81389e28 100644 --- a/Source/reactions/React_nd.F90 +++ b/Source/reactions/React_nd.F90 @@ -220,8 +220,7 @@ subroutine ca_react_state_simplified_sdc(lo,hi, & use network , only : nspec, naux use meth_params_module, only : NVAR, URHO, UMX, UMZ, UEDEN, UEINT, UTEMP, & - UFS, UFX, & - react_T_min, react_T_max, react_rho_min, react_rho_max + UFS, UFX #ifdef SHOCK_VAR use meth_params_module, only : USHK, disable_shock_burning #endif @@ -273,8 +272,7 @@ subroutine ca_react_state_simplified_sdc(lo,hi, & ! Don't burn if we're outside of the relevant (rho, T) range. - if (uold(i,j,k,UTEMP) < react_T_min .or. uold(i,j,k,UTEMP) > react_T_max .or. & - uold(i,j,k,URHO) < react_rho_min .or. uold(i,j,k,URHO) > react_rho_max) cycle + if (.not. okay_to_burn(U(i,j,k,:))) cycle ! Feed in the old-time state data. diff --git a/Source/reactions/react_util.F90 b/Source/reactions/react_util.F90 index bcf873469c..7c15718db2 100644 --- a/Source/reactions/react_util.F90 +++ b/Source/reactions/react_util.F90 @@ -5,6 +5,27 @@ module react_util_module contains + pure function okay_to_burn(state) result(burn_flag) + + use meth_params_module, only : NVAR, URHO, UTEMP, & + react_T_min, react_T_max, react_rho_min, react_rho_max + implicit none + + real(rt), intent(in) :: state(NVAR) + logical :: burn_flag + + burn_flag = .true. + + if (state(UTEMP) < react_T_min .or. state(UTEMP) > react_T_max .or. & + state(URHO) < react_rho_min .or. state(URHO) > react_rho_max) then + burn_flag = .false. + end if + + return + + end function okay_to_burn + + subroutine single_zone_react_source(state, R, i, j, k, burn_state) use burn_type_module, only : burn_t, net_ienuc From e578f065c99ed3e67e00272e2a67c8d0ff399ea4 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 19 Aug 2019 13:28:17 -0400 Subject: [PATCH 03/72] first pass at implementing limits on burning for SDC addresses #598 --- Source/driver/sdc_util.F90 | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 26df99a566..2dec54fbe4 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1177,6 +1177,7 @@ subroutine ca_sdc_update_centers_o4(lo, hi, dt_m, & ! m+1, which is dt_m = dt/2. use meth_params_module, only : NVAR + use react_util_module, only : okay_to_burn implicit none @@ -1198,9 +1199,14 @@ subroutine ca_sdc_update_centers_o4(lo, hi, dt_m, & do i = lo(1), hi(1) ! we come in with U_new being a guess for the updated solution + if (okay_to_burn(U_old(i,j,k,:))) then + call sdc_solve(dt_m, U_old(i,j,k,:), U_new(i,j,k,:), & + C(i,j,k,:), sdc_iteration) + else + ! no reactions, so it is a straightforward update + U_new(i,j,k,:) = U_old(i,j,k,:) + dt_m * C(i,j,k,:) - call sdc_solve(dt_m, U_old(i,j,k,:), U_new(i,j,k,:), C(i,j,k,:), sdc_iteration) - + end if enddo enddo enddo @@ -1277,7 +1283,11 @@ subroutine ca_instantaneous_react(lo, hi, & do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) - call single_zone_react_source(state(i,j,k,:), R_source(i,j,k,:), i,j,k, burn_state) + if (okay_to_burn(state(i,j,k,:))) then + call single_zone_react_source(state(i,j,k,:), R_source(i,j,k,:), i,j,k, burn_state) + else + R_source(i,j,k,:) = ZERO + end if enddo enddo enddo @@ -1307,8 +1317,6 @@ subroutine ca_store_reaction_state(lo, hi, & integer :: i, j, k - - do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) From 2c0e732eda716ac60ae161a3ee26af340c250d95 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 19 Aug 2019 19:21:14 -0400 Subject: [PATCH 04/72] a multilevel inputs file --- .../acoustic_pulse/inputs.2d.64.sdc_ml | 58 +++++++++++++++++++ Exec/hydro_tests/acoustic_pulse/probin | 3 + 2 files changed, 61 insertions(+) create mode 100644 Exec/hydro_tests/acoustic_pulse/inputs.2d.64.sdc_ml diff --git a/Exec/hydro_tests/acoustic_pulse/inputs.2d.64.sdc_ml b/Exec/hydro_tests/acoustic_pulse/inputs.2d.64.sdc_ml new file mode 100644 index 0000000000..f69dcf9798 --- /dev/null +++ b/Exec/hydro_tests/acoustic_pulse/inputs.2d.64.sdc_ml @@ -0,0 +1,58 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 500 +stop_time = 0.24 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 +geometry.coord_sys = 0 # 0 => cart +geometry.prob_lo = 0 0 +geometry.prob_hi = 1 1 +amr.n_cell = 64 64 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 0 0 +castro.hi_bc = 0 0 + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 0 + +castro.time_integration_method = 2 +castro.sdc_order = 2 + +# TIME STEP CONTROL +castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.cfl = 0.5 # cfl number for hyperbolic system +castro.init_shrink = 0.01 # scale back initial timestep +castro.change_max = 1.1 # maximum increase in dt over successive steps +castro.fixed_dt = 3.0e-3 + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp +#amr.grid_log = grdlog # name of grid logging file + +# REFINEMENT / REGRIDDING +amr.max_level = 2 # maximum level number allowed +amr.ref_ratio = 2 2 2 2 # refinement ratio +amr.regrid_int = 2 # how often to regrid +amr.blocking_factor = 8 # block factor in grid generation +amr.max_grid_size = 256 + +# CHECKPOINT FILES +amr.check_file = acoustic_pulse_ml_chk # root name of checkpoint file +amr.check_int = 100 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = acoustic_pulse_ml_plt +amr.plot_int = -1 +amr.plot_per = 0.24 +amr.derive_plot_vars=ALL + +# PROBIN FILENAME +amr.probin_file = probin diff --git a/Exec/hydro_tests/acoustic_pulse/probin b/Exec/hydro_tests/acoustic_pulse/probin index a03b1532da..dc8795fc7e 100644 --- a/Exec/hydro_tests/acoustic_pulse/probin +++ b/Exec/hydro_tests/acoustic_pulse/probin @@ -10,6 +10,9 @@ max_denerr_lev = 3 max_dengrad_lev = 3 + velerr = 0.02 + max_velerr_lev = 3 + presserr = 3 pressgrad = 0.01 max_presserr_lev = 3 From bc8e95c9387d2f53e263652bc48f88806ca17c78 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 20 Aug 2019 16:52:59 -0400 Subject: [PATCH 05/72] first pass at fixing the flux registers for SDC --- CHANGES | 7 +++++++ Source/driver/Castro.H | 1 + Source/driver/Castro.cpp | 1 + Source/driver/Castro_setup.cpp | 6 ++++++ Source/hydro/Castro_mol_hydro.cpp | 20 ++++++++++++++++---- 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 3d4d46e3af..5282a9296e 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,13 @@ recent changes -- The velerr tagging now takes the abs() of the velocity component to ensure we tag large positive and negative velocities. + -- The SDC algorithm now implements the burning conditionals + depending on rho and T (react_rho_min, react_rho_max, + react_T_min, react_T_max) + + -- The SDC/MOL PLM reconstruction now implements reflecting BCs on + the interface states + # 19.08.1 -- Fix CUDA compilation diff --git a/Source/driver/Castro.H b/Source/driver/Castro.H index 7377306759..d2b284ad06 100644 --- a/Source/driver/Castro.H +++ b/Source/driver/Castro.H @@ -1908,6 +1908,7 @@ protected: static int SDC_NODES; static amrex::Vector dt_sdc; + static amrex::Vector node_weights; /// diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp index 4b427440a6..4b319cde69 100644 --- a/Source/driver/Castro.cpp +++ b/Source/driver/Castro.cpp @@ -141,6 +141,7 @@ Vector Castro::c_mol; int Castro::SDC_NODES; Vector Castro::dt_sdc; +Vector Castro::node_weights; #ifdef AMREX_USE_CUDA int Castro::numBCThreadsMin[3] = {1, 1, 1}; diff --git a/Source/driver/Castro_setup.cpp b/Source/driver/Castro_setup.cpp index a825ea3081..19a7d0c4f8 100644 --- a/Source/driver/Castro_setup.cpp +++ b/Source/driver/Castro_setup.cpp @@ -1167,6 +1167,9 @@ Castro::variableSetUp () dt_sdc.resize(SDC_NODES); dt_sdc = {0.0, 1.0}; + node_weights.resize(SDC_NODES); + node_weights = {0.5, 0.5}; + } else if (sdc_order == 4) { SDC_NODES = 3; @@ -1174,6 +1177,9 @@ Castro::variableSetUp () dt_sdc.resize(SDC_NODES); dt_sdc = {0.0, 0.5, 1.0}; + node_weights.resize(SDC_NODES); + node_weights = {1.0/6.0, 4.0/6.0, 1.0/6.0}; + } else { amrex::Error("invalid value of sdc_order"); } diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index 441e39c3a1..e322b5b1cf 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -89,6 +89,10 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) stage_weight = b_mol[mol_iteration]; } + if (time_integration_method == SpectralDeferredCorrections) { + stage_weight = node_weights[current_sdc_node]; + } + #ifndef AMREX_USE_CUDA if (mol_order == 4 || sdc_order == 4) { @@ -490,7 +494,14 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // Store the fluxes from this advance -- we weight them by the // integrator weight for this stage - for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { + // For SDC, we store node 0 the only time we enter here (the + // first iteration) and we store the other nodes only on the + // last iteration. + if (time_integration_method == MethodOfLines || + (time_integration_method == SpectralDeferredCorrections && + (current_sdc_node == 0 || sdc_iteration == sdc_order+sdc_extra-1))) { + + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { Array4 const flux_fab = (flux[idir]).array(); Array4 fluxes_fab = (*fluxes[idir]).array(mfi); @@ -502,10 +513,10 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) fluxes_fab(i,j,k,n) += stage_weight * flux_fab(i,j,k,n); }); - } + } #if AMREX_SPACEDIM <= 2 - if (!Geom().IsCartesian()) { + if (!Geom().IsCartesian()) { Array4 pradial_fab = pradial.array(); Array4 P_radial_fab = P_radial.array(mfi); @@ -516,8 +527,9 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) P_radial_fab(i,j,k,0) += scale * pradial_fab(i,j,k,0); }); - } + } #endif + } } // MFIter loop From b3b34bcd8356462fb0271c86296126d841c21290 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 21 Aug 2019 20:32:05 -0400 Subject: [PATCH 06/72] add plm_well_balanced --- Source/driver/_cpp_parameters | 4 ++ Source/driver/meth_params.F90 | 56 +++++++++++-------- .../driver/param_includes/castro_defaults.H | 1 + .../param_includes/castro_job_info_tests.H | 1 + Source/driver/param_includes/castro_params.H | 1 + Source/driver/param_includes/castro_queries.H | 1 + 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Source/driver/_cpp_parameters b/Source/driver/_cpp_parameters index c64a3d8b4d..81216f08e0 100644 --- a/Source/driver/_cpp_parameters +++ b/Source/driver/_cpp_parameters @@ -106,6 +106,10 @@ ppm_reference_eigenvectors int 0 y # for piecewise linear, reconstruction order to use plm_iorder int 2 y +# for piecewise linear, MOL or SDC, do well-balanced reconstruction on +# pressure +plm_well_balanced int 0 y + # do we drop from our regular Riemann solver to HLL when we # are in shocks to avoid the odd-even decoupling instability? hybrid_riemann int 0 y diff --git a/Source/driver/meth_params.F90 b/Source/driver/meth_params.F90 index 9c15f02d45..4f5c74782a 100644 --- a/Source/driver/meth_params.F90 +++ b/Source/driver/meth_params.F90 @@ -132,6 +132,7 @@ module meth_params_module integer, allocatable, save :: ppm_predict_gammae integer, allocatable, save :: ppm_reference_eigenvectors integer, allocatable, save :: plm_iorder + integer, allocatable, save :: plm_well_balanced integer, allocatable, save :: hybrid_riemann integer, allocatable, save :: riemann_solver integer, allocatable, save :: cg_maxiter @@ -225,6 +226,7 @@ module meth_params_module attributes(managed) :: ppm_predict_gammae attributes(managed) :: ppm_reference_eigenvectors attributes(managed) :: plm_iorder +attributes(managed) :: plm_well_balanced attributes(managed) :: hybrid_riemann attributes(managed) :: riemann_solver attributes(managed) :: cg_maxiter @@ -351,6 +353,7 @@ module meth_params_module !$acc create(ppm_predict_gammae) & !$acc create(ppm_reference_eigenvectors) & !$acc create(plm_iorder) & + !$acc create(plm_well_balanced) & !$acc create(hybrid_riemann) & !$acc create(riemann_solver) & !$acc create(cg_maxiter) & @@ -558,6 +561,8 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para ppm_reference_eigenvectors = 0; allocate(plm_iorder) plm_iorder = 2; + allocate(plm_well_balanced) + plm_well_balanced = 0; allocate(hybrid_riemann) hybrid_riemann = 0; allocate(riemann_solver) @@ -714,6 +719,7 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para call pp%query("ppm_predict_gammae", ppm_predict_gammae) call pp%query("ppm_reference_eigenvectors", ppm_reference_eigenvectors) call pp%query("plm_iorder", plm_iorder) + call pp%query("plm_well_balanced", plm_well_balanced) call pp%query("hybrid_riemann", hybrid_riemann) call pp%query("riemann_solver", riemann_solver) call pp%query("cg_maxiter", cg_maxiter) @@ -786,30 +792,29 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para !$acc device(time_integration_method, limit_fourth_order, use_reconstructed_gamma1) & !$acc device(hybrid_hydro, ppm_type, ppm_temp_fix) & !$acc device(ppm_predict_gammae, ppm_reference_eigenvectors, plm_iorder) & - !$acc device(hybrid_riemann, riemann_solver, cg_maxiter) & - !$acc device(cg_tol, cg_blend, use_eos_in_riemann) & - !$acc device(riemann_speed_limit, use_flattening, transverse_use_eos) & - !$acc device(transverse_reset_density, transverse_reset_rhoe, dual_energy_eta1) & - !$acc device(dual_energy_eta2, use_pslope, limit_fluxes_on_small_dens) & - !$acc device(density_reset_method, allow_small_energy, do_sponge) & - !$acc device(sponge_implicit, first_order_hydro, hse_zero_vels) & - !$acc device(hse_interp_temp, hse_reflect_vels, mol_order) & - !$acc device(sdc_order, sdc_extra, sdc_solver) & - !$acc device(sdc_solver_tol_dens, sdc_solver_tol_spec, sdc_solver_tol_ener) & - !$acc device(sdc_solver_atol, sdc_solver_relax_factor, sdc_solve_for_rhoe) & - !$acc device(sdc_use_analytic_jac, cfl, dtnuc_e) & - !$acc device(dtnuc_X, dtnuc_X_threshold, do_react) & - !$acc device(react_T_min, react_T_max, react_rho_min) & - !$acc device(react_rho_max, disable_shock_burning, T_guess) & - !$acc device(diffuse_temp, diffuse_cutoff_density, diffuse_cutoff_density_hi) & - !$acc device(diffuse_cond_scale_fac, do_grav, grav_source_type) & - !$acc device(do_rotation, rot_period, rot_period_dot) & - !$acc device(rotation_include_centrifugal, rotation_include_coriolis, rotation_include_domegadt) & - !$acc device(state_in_rotating_frame, rot_source_type, implicit_rotation_update) & - !$acc device(rot_axis, use_point_mass, point_mass) & - !$acc device(point_mass_fix_solution, do_acc, grown_factor) & - !$acc device(track_grid_losses, const_grav) & - !$acc device(get_g_from_phi) + !$acc device(plm_well_balanced, hybrid_riemann, riemann_solver) & + !$acc device(cg_maxiter, cg_tol, cg_blend) & + !$acc device(use_eos_in_riemann, riemann_speed_limit, use_flattening) & + !$acc device(transverse_use_eos, transverse_reset_density, transverse_reset_rhoe) & + !$acc device(dual_energy_eta1, dual_energy_eta2, use_pslope) & + !$acc device(limit_fluxes_on_small_dens, density_reset_method, allow_small_energy) & + !$acc device(do_sponge, sponge_implicit, first_order_hydro) & + !$acc device(hse_zero_vels, hse_interp_temp, hse_reflect_vels) & + !$acc device(mol_order, sdc_order, sdc_extra) & + !$acc device(sdc_solver, sdc_solver_tol_dens, sdc_solver_tol_spec) & + !$acc device(sdc_solver_tol_ener, sdc_solver_atol, sdc_solver_relax_factor) & + !$acc device(sdc_solve_for_rhoe, sdc_use_analytic_jac, cfl) & + !$acc device(dtnuc_e, dtnuc_X, dtnuc_X_threshold) & + !$acc device(do_react, react_T_min, react_T_max) & + !$acc device(react_rho_min, react_rho_max, disable_shock_burning) & + !$acc device(T_guess, diffuse_temp, diffuse_cutoff_density) & + !$acc device(diffuse_cutoff_density_hi, diffuse_cond_scale_fac, do_grav) & + !$acc device(grav_source_type, do_rotation, rot_period) & + !$acc device(rot_period_dot, rotation_include_centrifugal, rotation_include_coriolis) & + !$acc device(rotation_include_domegadt, state_in_rotating_frame, rot_source_type) & + !$acc device(implicit_rotation_update, rot_axis, use_point_mass) & + !$acc device(point_mass, point_mass_fix_solution, do_acc) & + !$acc device(grown_factor, track_grid_losses, const_grav, get_g_from_phi) #ifdef GRAVITY @@ -954,6 +959,9 @@ subroutine ca_finalize_meth_params() bind(C, name="ca_finalize_meth_params") if (allocated(plm_iorder)) then deallocate(plm_iorder) end if + if (allocated(plm_well_balanced)) then + deallocate(plm_well_balanced) + end if if (allocated(hybrid_riemann)) then deallocate(hybrid_riemann) end if diff --git a/Source/driver/param_includes/castro_defaults.H b/Source/driver/param_includes/castro_defaults.H index e3b4651c45..520bf9975c 100644 --- a/Source/driver/param_includes/castro_defaults.H +++ b/Source/driver/param_includes/castro_defaults.H @@ -46,6 +46,7 @@ int Castro::ppm_temp_fix = 0; int Castro::ppm_predict_gammae = 0; int Castro::ppm_reference_eigenvectors = 0; int Castro::plm_iorder = 2; +int Castro::plm_well_balanced = 0; int Castro::hybrid_riemann = 0; int Castro::riemann_solver = 0; int Castro::cg_maxiter = 12; diff --git a/Source/driver/param_includes/castro_job_info_tests.H b/Source/driver/param_includes/castro_job_info_tests.H index 80b0af3c99..0d1978eab0 100644 --- a/Source/driver/param_includes/castro_job_info_tests.H +++ b/Source/driver/param_includes/castro_job_info_tests.H @@ -41,6 +41,7 @@ jobInfoFile << (Castro::ppm_temp_fix == 0 ? " " : "[*] ") << "castro.ppm_temp jobInfoFile << (Castro::ppm_predict_gammae == 0 ? " " : "[*] ") << "castro.ppm_predict_gammae = " << Castro::ppm_predict_gammae << std::endl; jobInfoFile << (Castro::ppm_reference_eigenvectors == 0 ? " " : "[*] ") << "castro.ppm_reference_eigenvectors = " << Castro::ppm_reference_eigenvectors << std::endl; jobInfoFile << (Castro::plm_iorder == 2 ? " " : "[*] ") << "castro.plm_iorder = " << Castro::plm_iorder << std::endl; +jobInfoFile << (Castro::plm_well_balanced == 0 ? " " : "[*] ") << "castro.plm_well_balanced = " << Castro::plm_well_balanced << std::endl; jobInfoFile << (Castro::hybrid_riemann == 0 ? " " : "[*] ") << "castro.hybrid_riemann = " << Castro::hybrid_riemann << std::endl; jobInfoFile << (Castro::riemann_solver == 0 ? " " : "[*] ") << "castro.riemann_solver = " << Castro::riemann_solver << std::endl; jobInfoFile << (Castro::cg_maxiter == 12 ? " " : "[*] ") << "castro.cg_maxiter = " << Castro::cg_maxiter << std::endl; diff --git a/Source/driver/param_includes/castro_params.H b/Source/driver/param_includes/castro_params.H index 96fc145b10..48fef0acf1 100644 --- a/Source/driver/param_includes/castro_params.H +++ b/Source/driver/param_includes/castro_params.H @@ -46,6 +46,7 @@ static int ppm_temp_fix; static int ppm_predict_gammae; static int ppm_reference_eigenvectors; static int plm_iorder; +static int plm_well_balanced; static int hybrid_riemann; static int riemann_solver; static int cg_maxiter; diff --git a/Source/driver/param_includes/castro_queries.H b/Source/driver/param_includes/castro_queries.H index ead8a6a29c..6ff45707d5 100644 --- a/Source/driver/param_includes/castro_queries.H +++ b/Source/driver/param_includes/castro_queries.H @@ -46,6 +46,7 @@ pp.query("ppm_temp_fix", ppm_temp_fix); pp.query("ppm_predict_gammae", ppm_predict_gammae); pp.query("ppm_reference_eigenvectors", ppm_reference_eigenvectors); pp.query("plm_iorder", plm_iorder); +pp.query("plm_well_balanced", plm_well_balanced); pp.query("hybrid_riemann", hybrid_riemann); pp.query("riemann_solver", riemann_solver); pp.query("cg_maxiter", cg_maxiter); From 974b26b0ce529c209e71750cfdf04823526febe0 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 21 Aug 2019 21:07:55 -0400 Subject: [PATCH 07/72] add first part of the Kappeli well-balanced scheme still need to modify the interface states with HSE --- Source/hydro/Castro_ctu_nd.F90 | 17 ++-- Source/hydro/Castro_mol_nd.F90 | 4 +- Source/hydro/slope_nd.F90 | 139 +++++++++++++++++++++++---------- 3 files changed, 110 insertions(+), 50 deletions(-) diff --git a/Source/hydro/Castro_ctu_nd.F90 b/Source/hydro/Castro_ctu_nd.F90 index 432d6bf777..f2c3634ecf 100644 --- a/Source/hydro/Castro_ctu_nd.F90 +++ b/Source/hydro/Castro_ctu_nd.F90 @@ -321,18 +321,19 @@ subroutine ctu_plm_states(lo, hi, & do n = 1, NQ if (.not. reconstruct_state(n)) cycle call uslope(lo, hi, idir, & - q, qd_lo, qd_hi, n, & - flatn, f_lo, f_hi, & - dq, dq_lo, dq_hi) + q, qd_lo, qd_hi, n, & + flatn, f_lo, f_hi, & + dq, dq_lo, dq_hi, & + domlo, domhi) end do if (use_pslope == 1) then call pslope(lo, hi, idir, & - q, qd_lo, qd_hi, & - flatn, f_lo, f_hi, & - dq, dq_lo, dq_hi, & - srcQ, src_lo, src_hi, & - dx) + q, qd_lo, qd_hi, & + flatn, f_lo, f_hi, & + dq, dq_lo, dq_hi, & + srcQ, src_lo, src_hi, & + dx) endif diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index 4a8afe5b0d..fc12cc7f2b 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -59,7 +59,8 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & call uslope(lo, hi, idir, & q, q_lo, q_hi, n, & flatn, fl_lo, fl_hi, & - dq, dq_lo, dq_hi) + dq, dq_lo, dq_hi, & + domlo, domhi) end do @@ -432,7 +433,6 @@ subroutine ca_mol_consup(lo, hi, & first_order_hydro, difmag, hybrid_riemann, & limit_fluxes_on_small_dens, ppm_type, ppm_temp_fix use amrex_constants_module, only : ZERO, HALF, ONE, FOURTH - use slope_module, only : uslope use amrex_fort_module, only : rt => amrex_real #ifdef HYBRID_MOMENTUM use hybrid_advection_module, only : add_hybrid_advection_source diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index f977e772a6..f0c599a8a9 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -13,11 +13,12 @@ module slope_module subroutine uslope(lo, hi, idir, & q, qd_lo, qd_hi, n, & flatn, f_lo, f_hi, & - dq, qpd_lo, qpd_hi) + dq, qpd_lo, qpd_hi, & + domlo, domhi) - use meth_params_module, only: NQ, plm_iorder + use meth_params_module, only: NQ, plm_iorder, QU use amrex_constants_module, only: ZERO, HALF, ONE, TWO, FOUR3RD, FOURTH, SIXTH - + use prob_params_module, only : Symmetry, physbc_lo, physbc_hi use amrex_fort_module, only : rt => amrex_real implicit none @@ -31,6 +32,7 @@ subroutine uslope(lo, hi, idir, & real(rt), intent(inout) :: dq(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),NQ) integer, intent(in) :: lo(3), hi(3) integer, intent(in), value :: idir + integer, intent(in) :: domlo(3), domhi(3) integer :: i, j, k @@ -38,6 +40,8 @@ subroutine uslope(lo, hi, idir, & real(rt) :: dlftp1, drgtp1, dfp1 real(rt) :: dlftm1, drgtm1, dfm1 + real(rt) :: qm2, qm1, q0, qp1, qp2 + !$gpu if (plm_iorder == 1) then @@ -58,49 +62,104 @@ subroutine uslope(lo, hi, idir, & do j = lo(2), hi(2) do i = lo(1), hi(1) - ! First compute Fromm slopes + if (plm_well_balanced == 1 .and. n == QPRES) then + ! we'll only do a second-order pressure slope, + ! but we'll follow the well-balanced scheme of + ! Kappeli. Note at the moment we are assuming + ! constant gravity. + p0 = q(i,j,k,QPRES) + pp1 = q(i+1,j,k,QPRES) - (p0 + HALF*dx(1)*(q(i,j,k,QRHO) + q(i+1,j,k,QRHO))*const_grav) + pm1 = q(i-1,j,k,QPRES) - (p0 - HALF*dx(1)*(q(i,j,k,QRHO) + q(i-1,j,k,QRHO))*const_grav) + + if (i == domlo(1) .and. physbc_lo(1) == Symmetry) then + pm1 = ZERO ! HSE is perfectly satisfied + end if + + if (i == domhi(1) .and. physbc_hi(1) == Symmetry) then + pp1 = ZERO + end if + + dlft = TWO*(p0 - pm1) + drgt = TWO*(pp1 - p0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - ! df at i+1 - dlftp1 = TWO*(q(i+1,j,k,n) - q(i,j,k,n)) - drgtp1 = TWO*(q(i+2,j,k,n) - q(i+1,j,k,n)) - dcen = FOURTH * (dlftp1 + drgtp1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftp1), abs(drgtp1)) - if (dlftp1*drgtp1 >= ZERO) then - dlim = slop else - dlim = ZERO - end if - dfp1 = dsgn*min(dlim, abs(dcen)) - ! df at i-1 - dlftm1 = TWO*(q(i-1,j,k,n) - q(i-2,j,k,n)) - drgtm1 = TWO*(q(i,j,k,n) - q(i-1,j,k,n)) - dcen = FOURTH * (dlftm1 + drgtm1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftm1), abs(drgtm1)) - if (dlftm1*drgtm1 >= ZERO) then - dlim = slop - else - dlim = ZERO - end if - dfm1 = dsgn*min(dlim, abs(dcen)) + ! the 4th order MC limiter + + qm2 = q(i-2,j,k,n) + qm1 = q(i-1,j,k,n) + q0 = q(i,j,k,n) + qp1 = q(i+1,j,k,n) + qp2 = q(i+2,j,k,n) + + ! special consideration for reflecting BCs -- see + ! Saltzmann p. 162 (but note that Saltzmann has a + ! sign error) + if (i == domlo(1) .and. n == QU .and. physbc_lo(1) == Symmetry) then + qm2 = -qp1 + qm1 = -3.0_rt*q0 + qp1 - 0.125_rt*(qp2 + qp1) + end if + + if (i == domhi(1) .and. n == QU .and. physbc_hi(1) == Symmetry) then + qp2 = -qm1 + qp1 = -3.0_rt*q0 + qm1 - 0.125_rt*(qm2 + qm1) + end if + + ! First compute Fromm slopes + + ! df at i+1 + dlftp1 = TWO*(qp1 - q0) + drgtp1 = TWO*(qp2 - qp1) + dcen = FOURTH * (dlftp1 + drgtp1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftp1), abs(drgtp1)) + if (dlftp1*drgtp1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfp1 = dsgn*min(dlim, abs(dcen)) + + ! df at i-1 + dlftm1 = TWO*(qm1 - qm2) + drgtm1 = TWO*(q0 - qm1) + dcen = FOURTH * (dlftm1 + drgtm1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftm1), abs(drgtm1)) + if (dlftm1*drgtm1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfm1 = dsgn*min(dlim, abs(dcen)) + + ! Now compute limited fourth order slopes at i + dlft = drgtm1 + drgt = dlftp1 + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) - ! Now compute limited fourth order slopes at i - dlft = drgtm1 - drgt = dlftp1 - dcen = FOURTH * (dlft + drgt) - dsgn = sign(ONE, dcen) - slop = min(abs(dlft), abs(drgt)) - if (dlft*drgt >= ZERO) then - dlim = slop - else - dlim = ZERO end if - dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) - dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) - end do end do end do From b259236cb36377f25a888ee9265efaadff31e4b4 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 22 Aug 2019 12:33:00 -0400 Subject: [PATCH 08/72] we are not using these custom BCs -- we use reflect now for 4th order --- .../bubble_convergence/bc_ext_fill_nd.F90 | 456 ------------------ 1 file changed, 456 deletions(-) delete mode 100644 Exec/reacting_tests/bubble_convergence/bc_ext_fill_nd.F90 diff --git a/Exec/reacting_tests/bubble_convergence/bc_ext_fill_nd.F90 b/Exec/reacting_tests/bubble_convergence/bc_ext_fill_nd.F90 deleted file mode 100644 index d83b0b4985..0000000000 --- a/Exec/reacting_tests/bubble_convergence/bc_ext_fill_nd.F90 +++ /dev/null @@ -1,456 +0,0 @@ -module bc_ext_fill_module - ! this module contains different routines for filling the - ! hydrodynamics boundary conditions - - ! .. note:: - ! the hydrostatic boundary conditions here rely on - ! constant gravity - - use amrex_constants_module, only: ZERO, HALF, ONE, TWO -#ifndef AMREX_USE_CUDA - use castro_error_module, only: castro_error -#endif - use amrex_fort_module, only: rt => amrex_real - use meth_params_module, only : NVAR, URHO, UMX, UMY, UMZ, & - UEDEN, UEINT, UFS, UTEMP, const_grav, & - hse_zero_vels, hse_interp_temp, hse_reflect_vels, & - xl_ext, xr_ext, yl_ext, yr_ext, zl_ext,zr_ext, EXT_HSE, EXT_INTERP - use prob_params_module, only: dim - - implicit none - - include 'AMReX_bc_types.fi' - -contains - - - subroutine ext_fill(lo, hi, adv, adv_lo, adv_hi, & - domlo, domhi, delta, xlo, time, bc) & - bind(C, name="ext_fill") - - use prob_params_module, only : problo, dim - use eos_module, only: eos - use eos_type_module, only: eos_t, eos_input_rt - use network, only: nspec - use model_parser_module, only: model_r, model_state, npts_model, idens_model, itemp_model, ispec_model, interpolate_sub - use amrex_filcc_module, only: amrex_filccn - use meth_params_module, only : sdc_order, mol_order - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: adv_lo(3), adv_hi(3) - integer, intent(in ) :: bc(dim,2,NVAR) - integer, intent(in ) :: domlo(3), domhi(3) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: adv(adv_lo(1):adv_hi(1),adv_lo(2):adv_hi(2),adv_lo(3):adv_hi(3),NVAR) - real(rt), intent(in ), value :: time - - integer :: i, j, k, q, iter, m, d - integer :: ioff, joff, koff - integer :: imin, imax, jmin, jmax, kmin, kmax - real(rt) :: x, y, yp, ym, z - - real(rt) :: rhoc, rhop, rhom, rho_avg - real(rt) :: Tc, Tp, Tm, T_avg - real(rt) :: rhoec, rhoep, rhoem, rhoe_avg - - real(rt) :: X_zone(nspec) - - type (eos_t) :: eos_state - - !$gpu - - - !------------------------------------------------------------------------- - ! x boundaries - !------------------------------------------------------------------------- - - ! XLO - if (bc(1,1,1) == EXT_DIR .and. lo(1) < domlo(1)) then - call castro_error("BCs not implemented for -X") - endif - - - ! XHI - if (bc(1,2,1) == EXT_DIR .and. hi(1) > domhi(1)) then - call castro_error("BCs not implemented for +X") - endif - - -#if AMREX_SPACEDIM >= 2 - !------------------------------------------------------------------------- - ! y boundaries - !------------------------------------------------------------------------- - - ! YLO - if (bc(2,1,1) == EXT_DIR .and. lo(2) < domlo(2)) then - - jmin = adv_lo(2) - jmax = domlo(2)-1 -#ifdef AMREX_USE_CUDA - if (hi(2) /= jmax) then - jmax = jmin - 1 - end if -#endif - do j = jmax, jmin, -1 - yp = problo(2) + delta(2)*(dble(j+1)+HALF) - y = problo(2) + delta(2)*(dble(j)+HALF) - ym = problo(2) + delta(2)*(dble(j-1)+HALF) - - do k = lo(3), hi(3) - do i = lo(1), hi(1) - - call interpolate_sub(rhop, yp, idens_model) - call interpolate_sub(rhoc, y, idens_model) - call interpolate_sub(rhom, ym, idens_model) - - call interpolate_sub(Tp, yp, itemp_model) - call interpolate_sub(Tc, y, itemp_model) - call interpolate_sub(Tm, ym, itemp_model) - - ! the composition in our initial model is uniform, so we just need - ! it at one point to get the average - do q = 1, nspec - call interpolate_sub(X_zone(q), y, ispec_model-1+q) - enddo - - ! extrap normal momentum - adv(i,j,k,UMY) = min(ZERO, adv(i,domlo(2),k,UMY)) - - ! zero transverse momentum - adv(i,j,k,UMX) = ZERO - adv(i,j,k,UMZ) = ZERO - - ! get the thermodynamics for the ym, y, yp - eos_state % rho = rhop - eos_state % T = Tp - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoep = rhop * eos_state % e - - eos_state % rho = rhoc - eos_state % T = Tc - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoec = rhoc * eos_state % e - - eos_state % rho = rhom - eos_state % T = Tm - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoem = rhom * eos_state % e - - ! now make the averages - if (sdc_order == 4 .or. mol_order == 4) then - rho_avg = rhoc + (ONE/24.0_rt) * (rhop - TWO*rhoc + rhom) - T_avg = Tc + (ONE/24.0_rt) * (Tp - TWO*Tc + Tm) - rhoe_avg = rhoec + (ONE/24.0_rt) * (rhoep - TWO*rhoec + rhoem) - else - rho_avg = rhoc - T_avg = Tc - rhoe_avg = rhoec - end if - - adv(i,j,k,URHO) = rho_avg - adv(i,j,k,UEINT) = rhoe_avg - adv(i,j,k,UTEMP) = T_avg - adv(i,j,k,UFS:UFS-1+nspec) = rho_avg*X_zone(:) - - ! this is an approximation -- the KE term here is only - ! second order accurate. We are relying on the fact - ! that for HSE, it should be much smaller than the - ! internal energy - adv(i,j,k,UEDEN) = rhoe_avg + & - HALF*sum(adv(i,j,k,UMX:UMZ)**2)/rhoc - - end do - end do - end do - - endif - - - ! YHI - if (bc(2,2,1) == EXT_DIR .and. hi(2) > domhi(2)) then - - ! interpolate thermodynamics from initial model - - jmin = domhi(2)+1 - jmax = adv_hi(2) -#ifdef AMREX_USE_CUDA - if (lo(2) /= jmin) then - jmin = jmax + 1 - end if -#endif - do j = jmin, jmax - yp = problo(2) + delta(2)*(dble(j+1)+HALF) - y = problo(2) + delta(2)*(dble(j)+HALF) - ym = problo(2) + delta(2)*(dble(j-1)+HALF) - - do k = lo(3), hi(3) - do i = lo(1), hi(1) - - call interpolate_sub(rhop, yp, idens_model) - call interpolate_sub(rhoc, y, idens_model) - call interpolate_sub(rhom, ym, idens_model) - - call interpolate_sub(Tp, yp, itemp_model) - call interpolate_sub(Tc, y, itemp_model) - call interpolate_sub(Tm, ym, itemp_model) - - ! the composition in our initial model is uniform, so we just need - ! it at one point to get the average - do q = 1, nspec - call interpolate_sub(X_zone(q), y, ispec_model-1+q) - enddo - - ! extrap normal momentum - adv(i,j,k,UMY) = max(ZERO, adv(i,domhi(2),k,UMY)) - - ! zero transverse momentum - adv(i,j,k,UMX) = ZERO - adv(i,j,k,UMZ) = ZERO - - ! get the thermodynamics for the ym, y, yp - eos_state % rho = rhop - eos_state % T = Tp - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoep = rhop * eos_state % e - - eos_state % rho = rhoc - eos_state % T = Tc - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoec = rhoc * eos_state % e - - eos_state % rho = rhom - eos_state % T = Tm - eos_state % xn(:) = X_zone - call eos(eos_input_rt, eos_state) - rhoem = rhom * eos_state % e - - ! now make the averages - if (sdc_order == 4 .or. mol_order == 4) then - rho_avg = rhoc + (ONE/24.0_rt) * (rhop - TWO*rhoc + rhom) - T_avg = Tc + (ONE/24.0_rt) * (Tp - TWO*Tc + Tm) - rhoe_avg = rhoec + (ONE/24.0_rt) * (rhoep - TWO*rhoec + rhoem) - else - rho_avg = rhoc - T_avg = Tc - rhoe_avg = rhoec - end if - - adv(i,j,k,URHO) = rho_avg - adv(i,j,k,UEINT) = rhoe_avg - adv(i,j,k,UTEMP) = T_avg - adv(i,j,k,UFS:UFS-1+nspec) = rho_avg*X_zone(:) - - ! this is an approximation -- the KE term here is only - ! second order accurate. We are relying on the fact - ! that for HSE, it should be much smaller than the - ! internal energy - adv(i,j,k,UEDEN) = rhoe_avg + & - HALF*sum(adv(i,j,k,UMX:UMZ)**2)/rhoc - - end do - end do - end do - - endif -#endif - -#if AMREX_SPACEDIM == 3 - - !------------------------------------------------------------------------- - ! z boundaries - !------------------------------------------------------------------------- - - ! ZLO - if (bc(3,1,1) == EXT_DIR .and. lo(3) < domlo(3)) then - call castro_error("BCs not implemented for -Z") - endif - - ! ZHI - if (bc(3,2,1) == EXT_DIR .and. hi(3) > domhi(3)) then - call castro_error("BCs not implemented for +Z") - end if -#endif - - end subroutine ext_fill - - - subroutine ext_denfill(lo, hi, adv, adv_lo, adv_hi, & - domlo, domhi, delta, xlo, time, bc) & - bind(C, name="ext_denfill") - - use prob_params_module, only: problo - use model_parser_module, only: npts_model, model_r, model_state, idens_model, interpolate_sub -#ifndef AMREX_USE_CUDA - use castro_error_module, only: castro_error -#endif - use amrex_filcc_module, only: amrex_filccn - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: adv_lo(3), adv_hi(3) - integer, intent(in ) :: bc(dim,2) - integer, intent(in ) :: domlo(3), domhi(3) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: adv(adv_lo(1):adv_hi(1),adv_lo(2):adv_hi(2),adv_lo(3):adv_hi(3)) - real(rt), intent(in ), value :: time - - integer :: i, j, k - integer :: jmin, jmax, kmin, kmax - real(rt) :: y, z - - !$gpu - - ! Note: this function should not be needed, technically, but is - ! provided to filpatch because there are many times in the algorithm - ! when just the density is needed. We try to rig up the filling so - ! that the same function is called here and in hypfill where all the - ! states are filled. - -#ifndef AMREX_USE_CUDA - ! XLO - if ( bc(1,1) == EXT_DIR .and. lo(1) < domlo(1)) then - call castro_error("We should not be here (xlo denfill)") - end if - - ! XHI - if ( bc(1,2) == EXT_DIR .and. hi(1) > domhi(1)) then - call castro_error("We should not be here (xhi denfill)") - endif -#endif - -#if AMREX_SPACEDIM >= 2 - ! YLO - if ( bc(2,1) == EXT_DIR .and. lo(2) < domlo(2)) then - jmin = adv_lo(2) - jmax = domlo(2)-1 -#ifdef AMREX_USE_CUDA - ! For CUDA, this should only be one thread doing the work: - ! we'll arbitrary choose the zone with index domlo(2) - 1. - if (hi(2) /= jmax) then - jmax = jmin - 1 - end if -#endif - do j = jmin, jmax - y = problo(2) + delta(2)*(dble(j) + HALF) - do k = lo(3), hi(3) - do i = lo(1), hi(1) - call interpolate_sub(adv(i,j,k), y, idens_model) - end do - end do - end do - end if - - ! YHI - if ( bc(2,2) == EXT_DIR .and. hi(2) > domhi(2)) then - jmin = domhi(2)+1 - jmax = adv_hi(2) -#ifdef AMREX_USE_CUDA - if (lo(2) /= jmin) then - jmin = jmax + 1 - end if -#endif - do j = jmin, jmax - y = problo(2) + delta(2)*(dble(j)+ HALF) - do k = lo(3), hi(3) - do i = lo(1), hi(1) - call interpolate_sub(adv(i,j,k), y, idens_model) - end do - end do - end do - end if -#endif - -#if AMREX_SPACEDIM == 3 - ! ZLO - if ( bc(3,1) == EXT_DIR .and. lo(3) < domlo(3)) then - - end if - - ! ZHI - if ( bc(3,2) == EXT_DIR .and. hi(3) > domhi(3)) then - - end if -#endif - - end subroutine ext_denfill - -#ifdef GRAVITY - subroutine ext_gravxfill(lo, hi, grav, grav_lo, grav_hi, & - domlo, domhi, delta, xlo, time, bc) & - bind(C, name="ext_gravxfill") - - use prob_params_module, only: problo - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: grav_lo(3), grav_hi(3) - integer, intent(in ) :: bc(dim,2) - integer, intent(in ) :: domlo(3), domhi(3) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: grav(grav_lo(1):grav_hi(1),grav_lo(2):grav_hi(2),grav_lo(3):grav_hi(3)) - real(rt), intent(in ), value :: time - - - !$gpu - - ! this is currently a stub - - end subroutine ext_gravxfill - - - subroutine ext_gravyfill(lo, hi, grav, grav_lo, grav_hi, & - domlo, domhi, delta, xlo, time, bc) & - bind(C, name="ext_gravyfill") - - use prob_params_module, only: problo - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: grav_lo(3), grav_hi(3) - integer, intent(in ) :: bc(dim,2) - integer, intent(in ) :: domlo(3), domhi(3) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: grav(grav_lo(1):grav_hi(1),grav_lo(2):grav_hi(2),grav_lo(3):grav_hi(3)) - real(rt), intent(in ), value :: time - - - !$gpu - - ! this is currently a stub - - end subroutine ext_gravyfill - - - subroutine ext_gravzfill(lo, hi, grav, grav_lo, grav_hi, & - domlo, domhi, delta, xlo, time, bc) & - bind(C, name="ext_gravzfill") - - use prob_params_module, only: problo - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: grav_lo(3), grav_hi(3) - integer, intent(in ) :: bc(dim,2) - integer, intent(in ) :: domlo(3), domhi(3) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: grav(grav_lo(1):grav_hi(1),grav_lo(2):grav_hi(2),grav_lo(3):grav_hi(3)) - real(rt), intent(in ), value :: time - - - !$gpu - - ! this is currently a stub - - end subroutine ext_gravzfill -#endif - -end module bc_ext_fill_module From e48d5c53e68d7027cc1a1f06e8a9e97ca033e6f4 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 22 Aug 2019 13:34:53 -0400 Subject: [PATCH 09/72] we don't use this anymore --- .../bubble_convergence/source_fill.F90 | 166 ------------------ 1 file changed, 166 deletions(-) delete mode 100644 Exec/reacting_tests/bubble_convergence/source_fill.F90 diff --git a/Exec/reacting_tests/bubble_convergence/source_fill.F90 b/Exec/reacting_tests/bubble_convergence/source_fill.F90 deleted file mode 100644 index 7093630032..0000000000 --- a/Exec/reacting_tests/bubble_convergence/source_fill.F90 +++ /dev/null @@ -1,166 +0,0 @@ -module source_fill_module - - use amrex_fort_module, only: rt => amrex_real - use meth_params_module, only: NVAR - - implicit none - - include 'AMReX_bc_types.fi' - -contains - - subroutine source_single_fill(lo, hi, state, s_lo, s_hi, domlo, domhi, delta, xlo, bc) bind(C, name="source_single_fill") - ! Used for a source fill of any StateData. - - use amrex_filcc_module, only: amrex_filccn - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: s_lo(3), s_hi(3) - integer, intent(in ) :: domlo(3), domhi(3) - integer, intent(in ) :: bc(AMREX_SPACEDIM, 2) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: state(s_lo(1):s_hi(1),s_lo(2):s_hi(2),s_lo(3):s_hi(3)) - - integer :: d - integer :: bc_temp(AMREX_SPACEDIM,2) - - !$gpu - - ! handle an external BC via extrapolation here -- user's can override as needed - bc_temp(:,:) = bc(:,:) - - do d = 1, AMREX_SPACEDIM - if (bc(d,1) == EXT_DIR .and. s_lo(d) < domlo(d)) then - bc_temp(d,1) = FOEXTRAP - end if - - if (bc(d,2) == EXT_DIR .and. s_hi(d) > domhi(d)) then - bc_temp(d,2) = FOEXTRAP - end if - end do - - call amrex_filccn(lo, hi, state, s_lo, s_hi, 1, domlo, domhi, delta, xlo, bc_temp) - - end subroutine source_single_fill - - - - subroutine source_multi_fill(lo, hi, state, s_lo, s_hi, domlo, domhi, delta, xlo, bc) bind(C, name="source_multi_fill") - - use meth_params_module, only : UMY, const_grav, mol_order, sdc_order - use amrex_filcc_module, only: amrex_filccn - use model_parser_module - use amrex_constants_module, only : HALF, ONE, TWO - use prob_params_module, only: problo - - implicit none - - integer, intent(in ) :: lo(3), hi(3) - integer, intent(in ) :: s_lo(3), s_hi(3) - integer, intent(in ) :: domlo(3), domhi(3) - integer, intent(in ) :: bc(AMREX_SPACEDIM, 2, NVAR) - real(rt), intent(in ) :: delta(3), xlo(3) - real(rt), intent(inout) :: state(s_lo(1):s_hi(1),s_lo(2):s_hi(2),s_lo(3):s_hi(3),NVAR) - - integer :: d, n - integer :: bc_temp(AMREX_SPACEDIM,2,NVAR) - real(rt) :: y, yp, ym, rhoc, rhop, rhom, rho_avg - integer :: i, j, k - integer :: jmin, jmax - - !$gpu - - - ! handle an external BC via extrapolation here -- user's can override as needed - bc_temp(:,:,:) = bc(:,:,:) - - do n = 1, NVAR - do d = 1, AMREX_SPACEDIM - if (bc(d,1,n) == EXT_DIR .and. s_lo(d) < domlo(d)) then - bc_temp(d,1,n) = FOEXTRAP - end if - - if (bc(d,2,n) == EXT_DIR .and. s_hi(d) > domhi(d)) then - bc_temp(d,2,n) = FOEXTRAP - end if - end do - end do - - call amrex_filccn(lo, hi, state, s_lo, s_hi, NVAR, domlo, domhi, delta, xlo, bc_temp) - - ! if we are UMY, we want to fill the gravity source term - ! explicitly so we maintain HSE. - ! YLO - - if (lo(2) < domlo(2) .and. bc(2,1,1) == EXT_DIR) then - jmin = s_lo(2) - jmax = domlo(2)-1 -#ifdef AMREX_USE_CUDA - ! For CUDA, this should only be one thread doing the work: - ! we'll arbitrary choose the zone with index domlo(2) - 1. - if (hi(2) /= jmax) then - jmax = jmin - 1 - end if -#endif - do j = jmin, jmax - yp = problo(2) + delta(2)*(dble(j+1) + HALF) - y = problo(2) + delta(2)*(dble(j) + HALF) - ym = problo(2) + delta(2)*(dble(j-1) + HALF) - - call interpolate_sub(rhop, yp, idens_model) - call interpolate_sub(rhoc, y, idens_model) - call interpolate_sub(rhom, ym, idens_model) - - do k = lo(3), hi(3) - do i = lo(1), hi(1) - - if (sdc_order == 4 .or. mol_order == 4) then - rho_avg = rhoc + (ONE/24.0_rt)*(rhop - TWO*rhoc + rhom) - state(i,j,k,UMY) = rho_avg * const_grav - else - state(i,j,k,UMY) = rhoc * const_grav - end if - - end do - end do - end do - end if - - ! YHI - if (hi(2) > domhi(2) .and. bc(2,2,1) == EXT_DIR) then - jmin = domhi(2)+1 - jmax = s_hi(2) -#ifdef AMREX_USE_CUDA - if (lo(2) /= jmin) then - jmin = jmax + 1 - end if -#endif - do j = jmin, jmax - yp = problo(2) + delta(2)*(dble(j+1) + HALF) - y = problo(2) + delta(2)*(dble(j) + HALF) - ym = problo(2) + delta(2)*(dble(j-1) + HALF) - - call interpolate_sub(rhop, yp, idens_model) - call interpolate_sub(rhoc, y, idens_model) - call interpolate_sub(rhom, ym, idens_model) - - do k = lo(3), hi(3) - do i = lo(1), hi(1) - - if (sdc_order == 4 .or. mol_order == 4) then - rho_avg = rhoc + (ONE/24.0_rt)*(rhop - TWO*rhoc + rhom) - state(i,j,k,UMY) = rho_avg * const_grav - else - state(i,j,k,UMY) = rhoc * const_grav - end if - - end do - end do - end do - end if - - end subroutine source_multi_fill - -end module source_fill_module From 2fe8fbc40e171064503066164d94712a4e516b00 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 22 Aug 2019 13:50:29 -0400 Subject: [PATCH 10/72] finish first pass at well-balanced. It kinda works, but not as good as pyro yet --- Source/hydro/Castro_ctu_nd.F90 | 2 +- Source/hydro/Castro_hydro_F.H | 1 + Source/hydro/Castro_mol_hydro.cpp | 1 + Source/hydro/Castro_mol_nd.F90 | 44 ++++++++++++++++++++++--------- Source/hydro/slope_nd.F90 | 21 ++++++++------- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Source/hydro/Castro_ctu_nd.F90 b/Source/hydro/Castro_ctu_nd.F90 index f2c3634ecf..2f9df48ab7 100644 --- a/Source/hydro/Castro_ctu_nd.F90 +++ b/Source/hydro/Castro_ctu_nd.F90 @@ -324,7 +324,7 @@ subroutine ctu_plm_states(lo, hi, & q, qd_lo, qd_hi, n, & flatn, f_lo, f_hi, & dq, dq_lo, dq_hi, & - domlo, domhi) + dx, domlo, domhi) end do if (use_pslope == 1) then diff --git a/Source/hydro/Castro_hydro_F.H b/Source/hydro/Castro_hydro_F.H index 4ff6d6f703..17546fc893 100644 --- a/Source/hydro/Castro_hydro_F.H +++ b/Source/hydro/Castro_hydro_F.H @@ -414,6 +414,7 @@ extern "C" BL_FORT_FAB_ARG_3D(dq), BL_FORT_FAB_ARG_3D(qm), BL_FORT_FAB_ARG_3D(qp), + const amrex::Real* dx, const int* domlo, const int* domhi); void ca_mol_ppm_reconstruct diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index e322b5b1cf..aa882faf5e 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -228,6 +228,7 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) BL_TO_FORTRAN_ANYD(dq), BL_TO_FORTRAN_ANYD(qm), BL_TO_FORTRAN_ANYD(qp), + AMREX_REAL_ANYD(dx), AMREX_INT_ANYD(domain_lo), AMREX_INT_ANYD(domain_hi)); } else { diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index fc12cc7f2b..c3d7282715 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -13,15 +13,15 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & dq, dq_lo, dq_hi, & qm, qm_lo, qm_hi, & qp, qp_lo, qp_hi, & - domlo, domhi) bind(C, name="ca_mol_plm_reconstruct") + dx, domlo, domhi) bind(C, name="ca_mol_plm_reconstruct") use castro_error_module use meth_params_module, only : NQ, NVAR, NGDNV, GDPRES, & UTEMP, UMX, & - use_flattening, QPRES, & + use_flattening, plm_well_balanced, QPRES, & QTEMP, QFS, QFX, QREINT, QRHO, QU, QV, QW, & first_order_hydro, hybrid_riemann, & - ppm_temp_fix + ppm_temp_fix, const_grav use amrex_constants_module, only : ZERO, HALF, ONE, FOURTH use slope_module, only : uslope use amrex_fort_module, only : rt => amrex_real @@ -45,6 +45,7 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & real(rt), intent(inout) :: qm(qm_lo(1):qm_hi(1), qm_lo(2):qm_hi(2), qm_lo(3):qm_hi(3), NQ, AMREX_SPACEDIM) real(rt), intent(inout) :: qp(qp_lo(1):qp_hi(1), qp_lo(2):qp_hi(2), qp_lo(3):qp_hi(3), NQ, AMREX_SPACEDIM) integer, intent(in) :: domlo(3), domhi(3) + real(rt), intent(in) :: dx(3) integer :: idir, i, j, k, n type (eos_t) :: eos_state @@ -60,7 +61,7 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & q, q_lo, q_hi, n, & flatn, fl_lo, fl_hi, & dq, dq_lo, dq_hi, & - domlo, domhi) + dx, domlo, domhi) end do @@ -68,20 +69,39 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & ! for each slope, fill the two adjacent edge states if (idir == 1) then - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - ! left state at i+1/2 interface - qm(i+1,j,k,n,1) = q(i,j,k,n) + HALF*dq(i,j,k,n) + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then - ! right state at i-1/2 interface - qp(i,j,k,n,1) = q(i,j,k,n) - HALF*dq(i,j,k,n) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + ! left state at i+1/2 interface + qm(i+1,j,k,n,1) = q(i,j,k,n) + HALF*dq(i,j,k,n) + & + HALF*dx(1)*q(i,j,k,QRHO)*const_grav + + ! right state at i-1/2 interface + qp(i,j,k,n,1) = q(i,j,k,n) - HALF*dq(i,j,k,n) - & + HALF*dx(1)*q(i,j,k,QRHO)*const_grav + end do end do end do - end do + else + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + ! left state at i+1/2 interface + qm(i+1,j,k,n,1) = q(i,j,k,n) + HALF*dq(i,j,k,n) + + ! right state at i-1/2 interface + qp(i,j,k,n,1) = q(i,j,k,n) - HALF*dq(i,j,k,n) + + end do + end do + end do + end if #if BL_SPACEDIM >= 2 else if (idir == 2) then do k = lo(3), hi(3) diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index f0c599a8a9..7577d92055 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -14,9 +14,10 @@ subroutine uslope(lo, hi, idir, & q, qd_lo, qd_hi, n, & flatn, f_lo, f_hi, & dq, qpd_lo, qpd_hi, & - domlo, domhi) + dx, domlo, domhi) - use meth_params_module, only: NQ, plm_iorder, QU + use meth_params_module, only: NQ, plm_iorder, QU, QPRES, QRHO, & + const_grav, plm_well_balanced use amrex_constants_module, only: ZERO, HALF, ONE, TWO, FOUR3RD, FOURTH, SIXTH use prob_params_module, only : Symmetry, physbc_lo, physbc_hi use amrex_fort_module, only : rt => amrex_real @@ -32,6 +33,7 @@ subroutine uslope(lo, hi, idir, & real(rt), intent(inout) :: dq(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),NQ) integer, intent(in) :: lo(3), hi(3) integer, intent(in), value :: idir + real(rt), intent(in) :: dx(3) integer, intent(in) :: domlo(3), domhi(3) integer :: i, j, k @@ -41,6 +43,7 @@ subroutine uslope(lo, hi, idir, & real(rt) :: dlftm1, drgtm1, dfm1 real(rt) :: qm2, qm1, q0, qp1, qp2 + real(rt) :: pp1, p0, pm1 !$gpu @@ -62,7 +65,7 @@ subroutine uslope(lo, hi, idir, & do j = lo(2), hi(2) do i = lo(1), hi(1) - if (plm_well_balanced == 1 .and. n == QPRES) then + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then ! we'll only do a second-order pressure slope, ! but we'll follow the well-balanced scheme of ! Kappeli. Note at the moment we are assuming @@ -71,13 +74,13 @@ subroutine uslope(lo, hi, idir, & pp1 = q(i+1,j,k,QPRES) - (p0 + HALF*dx(1)*(q(i,j,k,QRHO) + q(i+1,j,k,QRHO))*const_grav) pm1 = q(i-1,j,k,QPRES) - (p0 - HALF*dx(1)*(q(i,j,k,QRHO) + q(i-1,j,k,QRHO))*const_grav) - if (i == domlo(1) .and. physbc_lo(1) == Symmetry) then - pm1 = ZERO ! HSE is perfectly satisfied - end if + ! if (i == domlo(1) .and. physbc_lo(1) == Symmetry) then + ! pm1 = ZERO ! HSE is perfectly satisfied + ! end if - if (i == domhi(1) .and. physbc_hi(1) == Symmetry) then - pp1 = ZERO - end if + ! if (i == domhi(1) .and. physbc_hi(1) == Symmetry) then + ! pp1 = ZERO + ! end if dlft = TWO*(p0 - pm1) drgt = TWO*(pp1 - p0) From 17e5576d8f2319e34ba6dabd3471b1293df18916 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 13:06:08 -0400 Subject: [PATCH 11/72] fix p0 in well-balanced --- Source/hydro/slope_nd.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index 7577d92055..752674a66d 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -70,7 +70,7 @@ subroutine uslope(lo, hi, idir, & ! but we'll follow the well-balanced scheme of ! Kappeli. Note at the moment we are assuming ! constant gravity. - p0 = q(i,j,k,QPRES) + p0 = ZERO pp1 = q(i+1,j,k,QPRES) - (p0 + HALF*dx(1)*(q(i,j,k,QRHO) + q(i+1,j,k,QRHO))*const_grav) pm1 = q(i-1,j,k,QPRES) - (p0 - HALF*dx(1)*(q(i,j,k,QRHO) + q(i-1,j,k,QRHO))*const_grav) From 285f734935d1400e70da809e7366c6ef7f7de6ce Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 13:06:50 -0400 Subject: [PATCH 12/72] implement 1-d X BCs --- Source/problems/bc_ext_fill_nd.F90 | 38 ++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/Source/problems/bc_ext_fill_nd.F90 b/Source/problems/bc_ext_fill_nd.F90 index c718919c3b..b73287986f 100644 --- a/Source/problems/bc_ext_fill_nd.F90 +++ b/Source/problems/bc_ext_fill_nd.F90 @@ -950,8 +950,8 @@ subroutine ext_denfill(lo, hi, adv, adv_lo, adv_hi, & real(rt), intent(in ), value :: time integer :: i, j, k - integer :: jmin, jmax, kmin, kmax - real(rt) :: y, z + integer :: imin, imax, jmin, jmax, kmin, kmax + real(rt) :: x, y, z !$gpu @@ -964,12 +964,42 @@ subroutine ext_denfill(lo, hi, adv, adv_lo, adv_hi, & #ifndef AMREX_USE_CUDA ! XLO if ( bc(1,1) == EXT_DIR .and. lo(1) < domlo(1)) then - call castro_error("We should not be here (xlo denfill)") + imin = adv_lo(1) + imax = domlo(1)-1 +#ifdef AMREX_USE_CUDA + ! For CUDA, this should only be one thread doing the work: + ! we'll arbitrary choose the zone with index domlo(2) - 1. + if (hi(1) /= imax) then + imax = imin - 1 + end if +#endif + do i = imin, imax + x = problo(1) + delta(1)*(dble(i) + HALF) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + call interpolate_sub(adv(i,j,k), x, idens_model) + end do + end do + end do end if ! XHI if ( bc(1,2) == EXT_DIR .and. hi(1) > domhi(1)) then - call castro_error("We should not be here (xhi denfill)") + imin = domhi(1)+1 + imax = adv_hi(1) +#ifdef AMREX_USE_CUDA + if (lo(1) /= imin) then + imin = imax + 1 + end if +#endif + do i = imin, imax + x = problo(1) + delta(1)*(dble(i)+ HALF) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + call interpolate_sub(adv(i,j,k), x, idens_model) + end do + end do + end do endif #endif From 4fa3ed78a94757080de6d7e3914d34fa0fa3f4ad Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 14:08:40 -0400 Subject: [PATCH 13/72] add a +X HSE BC --- Source/problems/bc_ext_fill_nd.F90 | 167 ++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/Source/problems/bc_ext_fill_nd.F90 b/Source/problems/bc_ext_fill_nd.F90 index b73287986f..11cdea340a 100644 --- a/Source/problems/bc_ext_fill_nd.F90 +++ b/Source/problems/bc_ext_fill_nd.F90 @@ -47,8 +47,9 @@ subroutine ext_fill(lo, hi, adv, adv_lo, adv_hi, & integer :: ioff, joff, koff integer :: imin, imax, jmin, jmax, kmin, kmax real(rt) :: x, y, z - real(rt) :: dens_above, dens_base, temp_above - real(rt) :: pres_above, p_want, pres_zone, A + real(rt) :: dens_above, pres_above, temp_above + real(rt) :: dens_below, pres_below, temp_below + real(rt) :: dens_base, p_want, pres_zone, A real(rt) :: drho, dpdr, temp_zone, eint, X_zone(nspec), dens_zone integer, parameter :: MAX_ITER = 250 @@ -289,10 +290,170 @@ subroutine ext_fill(lo, hi, adv, adv_lo, adv_hi, & if (bc(1,2,1) == EXT_DIR .and. hi(1) > domhi(1)) then if (xr_ext == EXT_HSE) then + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + ! we are integrating along a column at constant i. + ! Make sure that our starting state is well-defined + dens_below = adv(domhi(1),j,k,URHO) + + ! sometimes, we might be working in a corner + ! where the ghost cells above us have not yet + ! been initialized. In that case, take the info + ! from the initial model + if (dens_below == ZERO) then + x = problo(1) + delta(1)*(dble(domhi(1)) + HALF) + + call interpolate_sub(dens_below, x, idens_model) + call interpolate_sub(temp_below, x, itemp_model) + + do m = 1, nspec + call interpolate_sub(X_zone(m), x, ispec_model-1+m) + end do + + else + temp_below = adv(domhi(1),j,k,UTEMP) + X_zone(:) = adv(domhi(1),j,k,UFS:UFS-1+nspec)/dens_below + endif + + ! keep track of the density at the base of the domain + dens_base = dens_below + + ! get pressure in this zone (the initial below zone) + eos_state%rho = dens_below + eos_state%T = temp_below + eos_state%xn(:) = X_zone(:) + + call eos(eos_input_rt, eos_state) + + eint = eos_state%e + pres_below = eos_state%p + + ! integrate downward + imin = domhi(1)+1 + imax = adv_hi(1) +#ifdef AMREX_USE_CUDA + ! For CUDA, this should only be one thread doing the work: + ! we'll arbitrary choose the zone with index domlo(1) - 1. + if (hi(1) /= imax) then + imax = imin - 1 + end if +#endif + do i = imin, imax + x = problo(1) + delta(1)*(dble(i) + HALF) + + ! HSE integration to get density, pressure + + ! initial guesses + dens_zone = dens_below + + ! temperature and species held constant in BCs + if (hse_interp_temp == 1) then + call interpolate_sub(temp_zone, x, itemp_model) + else + temp_zone = temp_below + endif + + converged_hse = .FALSE. + + do iter = 1, MAX_ITER + + ! pressure needed from HSE + p_want = pres_below + & + delta(1)*HALF*(dens_zone + dens_below)*const_grav + + ! pressure from EOS + eos_state%rho = dens_zone + eos_state%T = temp_zone + eos_state%xn(:) = X_zone(:) + + call eos(eos_input_rt, eos_state) + + pres_zone = eos_state%p + dpdr = eos_state%dpdr + eint = eos_state%e + + ! Newton-Raphson - we want to zero A = p_want - p(rho) + A = p_want - pres_zone + drho = A/(dpdr - HALF*delta(1)*const_grav) + + dens_zone = max(0.9_rt*dens_zone, & + min(dens_zone + drho, 1.1_rt*dens_zone)) + + ! convergence? + if (abs(drho) < TOL*dens_zone) then + converged_hse = .TRUE. + exit + endif + + enddo + #ifndef AMREX_USE_CUDA - call castro_error("ERROR: HSE boundaries not implemented for +X") + if (.not. converged_hse) then + print *, "i, j, k, domhi(1): ", i, j, k, domhi(1) + print *, "p_want: ", p_want + print *, "dens_zone: ", dens_zone + print *, "temp_zone: ", temp_zone + print *, "drho: ", drho + print *, " " + print *, "column info: " + print *, " dens: ", adv(i:domhi(1),j,k,URHO) + print *, " temp: ", adv(i:domhi(1),j,k,UTEMP) + call castro_error("ERROR in bc_ext_fill_nd: failure to converge in +X BC") + endif #endif + ! velocity + if (hse_zero_vels == 1) then + + ! zero normal momentum causes pi waves to pass through + adv(i,j,k,UMX) = ZERO + adv(i,j,k,UMY) = ZERO + adv(i,j,k,UMZ) = ZERO + + else + + if (hse_reflect_vels == 1) then + ! reflect normal, zero gradient for transverse + ! note: we need to match the corresponding + ! zone on the other side of the interface + ioff = i-domhi(1)-1 + adv(i,j,k,UMX) = -dens_zone*(adv(domhi(1)-ioff,j,k,UMX)/adv(domhi(1)-ioff,j,k,URHO)) + + adv(i,j,k,UMY) = -dens_zone*(adv(domhi(1),j,k,UMY)/dens_base) + adv(i,j,k,UMZ) = -dens_zone*(adv(domhi(1),j,k,UMZ)/dens_base) + else + ! zero gradient + adv(i,j,k,UMX) = dens_zone*(adv(domhi(1),j,k,UMX)/dens_base) + adv(i,j,k,UMY) = dens_zone*(adv(domhi(1),j,k,UMY)/dens_base) + adv(i,j,k,UMZ) = dens_zone*(adv(domhi(1),j,k,UMZ)/dens_base) + endif + endif + eos_state%rho = dens_zone + eos_state%T = temp_zone + eos_state%xn(:) = X_zone + + call eos(eos_input_rt, eos_state) + + pres_zone = eos_state%p + eint = eos_state%e + + ! store the final state + adv(i,j,k,URHO) = dens_zone + adv(i,j,k,UEINT) = dens_zone*eint + adv(i,j,k,UEDEN) = dens_zone*eint + & + HALF*sum(adv(i,j,k,UMX:UMZ)**2)/dens_zone + adv(i,j,k,UTEMP) = temp_zone + adv(i,j,k,UFS:UFS-1+nspec) = dens_zone*X_zone(:) + + ! for the next zone + dens_below = dens_zone + pres_below = pres_zone + + end do + end do + end do + elseif (xr_ext == EXT_INTERP) then ! interpolate thermodynamics from initial model From bdb75deb5a7e8b766b753dc81ab95a51eade5c84 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 14:10:04 -0400 Subject: [PATCH 14/72] add 2nd order MC slope option --- Source/driver/_cpp_parameters | 3 ++- Source/hydro/slope_nd.F90 | 33 ++++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Source/driver/_cpp_parameters b/Source/driver/_cpp_parameters index 81216f08e0..95d04d1eeb 100644 --- a/Source/driver/_cpp_parameters +++ b/Source/driver/_cpp_parameters @@ -104,7 +104,8 @@ ppm_predict_gammae int 0 y ppm_reference_eigenvectors int 0 y # for piecewise linear, reconstruction order to use -plm_iorder int 2 y +# 1 = piecewise constant, 2 = MC limiter, >2 = 4th order MC limiter +plm_iorder int 3 y # for piecewise linear, MOL or SDC, do well-balanced reconstruction on # pressure diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index 752674a66d..c1354ebf10 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -74,13 +74,13 @@ subroutine uslope(lo, hi, idir, & pp1 = q(i+1,j,k,QPRES) - (p0 + HALF*dx(1)*(q(i,j,k,QRHO) + q(i+1,j,k,QRHO))*const_grav) pm1 = q(i-1,j,k,QPRES) - (p0 - HALF*dx(1)*(q(i,j,k,QRHO) + q(i-1,j,k,QRHO))*const_grav) - ! if (i == domlo(1) .and. physbc_lo(1) == Symmetry) then - ! pm1 = ZERO ! HSE is perfectly satisfied - ! end if + if (i == domlo(1) .and. physbc_lo(1) == Symmetry) then + pm1 = ZERO ! HSE is perfectly satisfied + end if - ! if (i == domhi(1) .and. physbc_hi(1) == Symmetry) then - ! pp1 = ZERO - ! end if + if (i == domhi(1) .and. physbc_hi(1) == Symmetry) then + pp1 = ZERO + end if dlft = TWO*(p0 - pm1) drgt = TWO*(pp1 - p0) @@ -95,8 +95,27 @@ subroutine uslope(lo, hi, idir, & dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - else + else if (plm_iorder == 2) then + ! the 2nd order MC limiter + + qm1 = q(i-1,j,k,n) + q0 = q(i,j,k,n) + qp1 = q(i+1,j,k,n) + dlft = TWO*(q0 - qm1) + drgt = TWO*(qp1 - q0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) + + else ! the 4th order MC limiter qm2 = q(i-2,j,k,n) From d3a7ba365fd5e7bfd43411385d6ab1761a7cd324 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 15:11:48 -0400 Subject: [PATCH 15/72] fix comment --- Source/problems/bc_ext_fill_nd.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/problems/bc_ext_fill_nd.F90 b/Source/problems/bc_ext_fill_nd.F90 index 11cdea340a..6c8c0c2f37 100644 --- a/Source/problems/bc_ext_fill_nd.F90 +++ b/Source/problems/bc_ext_fill_nd.F90 @@ -329,7 +329,7 @@ subroutine ext_fill(lo, hi, adv, adv_lo, adv_hi, & eint = eos_state%e pres_below = eos_state%p - ! integrate downward + ! integrate upward imin = domhi(1)+1 imax = adv_hi(1) #ifdef AMREX_USE_CUDA From 80792a16469376874abe6645713c956ad76eb111 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 16:32:53 -0400 Subject: [PATCH 16/72] some 2nd order model support for well balancing --- .../bubble_convergence/initial_model.F90 | 217 ++++++++++++------ 1 file changed, 148 insertions(+), 69 deletions(-) diff --git a/Exec/reacting_tests/bubble_convergence/initial_model.F90 b/Exec/reacting_tests/bubble_convergence/initial_model.F90 index 062eb150a2..4cd78d603a 100644 --- a/Exec/reacting_tests/bubble_convergence/initial_model.F90 +++ b/Exec/reacting_tests/bubble_convergence/initial_model.F90 @@ -29,7 +29,7 @@ subroutine generate_initial_model(nx, xmin, xmax, model_params, nbuf) use eos_module, only: eos use eos_type_module, only: eos_t, eos_input_rt, eos_input_ps use network, only : nspec - use meth_params_module, only : const_grav + use meth_params_module, only : const_grav, sdc_order use amrex_paralleldescriptor_module, only: parallel_IOProcessor => amrex_pd_ioprocessor @@ -49,6 +49,12 @@ subroutine generate_initial_model(nx, xmin, xmax, model_params, nbuf) real(rt) :: pnew, p, s, rho, T, xn(nspec) real(rt) :: dx + logical :: converged_hse + real(rt) :: drho, p_want, dens_zone + integer :: iter + integer, parameter :: MAX_ITER = 100 + real(rt), parameter :: TOL = 1.e-10_rt + ! allocate the storage in the model_parser_module npts_model = nx + 2*nbuf @@ -81,97 +87,170 @@ subroutine generate_initial_model(nx, xmin, xmax, model_params, nbuf) p = eos_state % p - ! do RK 4 integration up from the lower boundary - do i = ibase, itop + nbuf - ! rho and T here are guesses for the EOS call - if (i == ibase) then - rho = eos_state % rho - T = eos_state % T - else - rho = model_state(i-1, idens_model) - T = model_state(i-1, itemp_model) - end if + if (sdc_order == 4) then + + ! a fourth order accurate method + + ! do RK 4 integration up from the lower boundary + do i = ibase, itop + nbuf + + ! rho and T here are guesses for the EOS call + if (i == ibase) then + rho = eos_state % rho + T = eos_state % T + else + rho = model_state(i-1, idens_model) + T = model_state(i-1, itemp_model) + end if + + xn(:) = model_params % xn(:) + + ! step size + if (i == ibase) then + h = HALF*dx + else + h = dx + end if + + ! entropy never changes in this model + s = entropy_fixed + + k1 = f(p, s, const_grav, rho, T, xn) + k2 = f(p + HALF*h*k1, s, const_grav, rho, T, xn) + k3 = f(p + HALF*h*k2, s, const_grav, rho, T, xn) + k4 = f(p + h*k3, s, const_grav, rho, T, xn) + + pnew = p + SIXTH*h*(k1 + TWO*k2 + TWO*k3 + k4) + + ! call the EOS to get the remainder of the thermodynamics + eos_state % T = T ! initial guess + eos_state % rho = rho ! initial guess + eos_state % xn(:) = model_params % xn(:) + eos_state % p = pnew + eos_state % s = s + + call eos(eos_input_ps, eos_state) + + ! update the thermodynamics in this zone + model_state(i, idens_model) = eos_state % rho + model_state(i, itemp_model) = eos_state % T + model_state(i, ipres_model) = eos_state % p + model_state(i, ispec_model:ispec_model-1+nspec) = eos_state % xn(:) + + ! reset for the next iteration + p = pnew + + enddo + + p = model_state(ibase, ipres_model) + + ! now integrate down + do i = ibase-1, 1, -1 + + rho = model_state(i+1, idens_model) + T = model_state(i+1, itemp_model) + xn(:) = model_params % xn(:) + + h = -dx + + ! entropy never changes in this model + s = entropy_fixed + + k1 = f(p, s, const_grav, rho, T, xn) + k2 = f(p + HALF*h*k1, s, const_grav, rho, T, xn) + k3 = f(p + HALF*h*k2, s, const_grav, rho, T, xn) + k4 = f(p + h*k3, s, const_grav, rho, T, xn) + + pnew = p + SIXTH*h*(k1 + TWO*k2 + TWO*k3 + k4) + + ! call the EOS to get the remainder of the thermodynamics + eos_state % T = T ! initial guess + eos_state % rho = rho ! initial guess + eos_state % xn(:) = model_params % xn(:) + eos_state % p = pnew + eos_state % s = s + + call eos(eos_input_ps, eos_state) + + ! update the thermodynamics in this zone + model_state(i, idens_model) = eos_state % rho + model_state(i, itemp_model) = eos_state % T + model_state(i, ipres_model) = eos_state % p + model_state(i, ispec_model:ispec_model-1+nspec) = eos_state % xn(:) + + ! reset for the next iteration + p = pnew - xn(:) = model_params % xn(:) + enddo - ! step size - if (i == ibase) then - h = HALF*dx - else - h = dx - end if + else - ! entropy never changes in this model - s = entropy_fixed + model_state(:, :) = ZERO - k1 = f(p, s, const_grav, rho, T, xn) - k2 = f(p + HALF*h*k1, s, const_grav, rho, T, xn) - k3 = f(p + HALF*h*k2, s, const_grav, rho, T, xn) - k4 = f(p + h*k3, s, const_grav, rho, T, xn) + ! a second order accurate scheme + do i = ibase, itop + nbuf - pnew = p + SIXTH*h*(k1 + TWO*k2 + TWO*k3 + k4) + if (i == ibase) then + model_state(i, idens_model) = model_params % dens_base + model_state(i, ipres_model) = p + HALF*dx* model_params % dens_base * const_grav - ! call the EOS to get the remainder of the thermodynamics - eos_state % T = T ! initial guess - eos_state % rho = rho ! initial guess - eos_state % xn(:) = model_params % xn(:) - eos_state % p = pnew - eos_state % s = s + eos_state % rho = model_state(i, idens_model) + eos_state % T = model_params % dens_base + eos_state % p = model_state(i, ipres_model) + eos_state % xn(:) = model_params % xn(:) + eos_state % s = entropy_fixed - call eos(eos_input_ps, eos_state) + call eos(eos_input_ps, eos_state) - ! update the thermodynamics in this zone - model_state(i, idens_model) = eos_state % rho - model_state(i, itemp_model) = eos_state % T - model_state(i, ipres_model) = eos_state % p - model_state(i, ispec_model:ispec_model-1+nspec) = eos_state % xn(:) + model_state(i, idens_model) = eos_state % rho + model_state(i, itemp_model) = eos_state % T + model_state(i, ispec_model:ispec_model-1+nspec) = xn(:) - ! reset for the next iteration - p = pnew + else - enddo + dens_zone = model_state(i-1, idens_model) - p = model_state(ibase, ipres_model) + converged_hse = .FALSE. - ! now integrate down - do i = ibase-1, 1, -1 + do iter = 1, MAX_ITER - rho = model_state(i+1, idens_model) - T = model_state(i+1, itemp_model) - xn(:) = model_params % xn(:) + p_want = model_state(i-1, ipres_model) + & + dx*HALF*(dens_zone + model_state(i-1, idens_model))*const_grav - h = -dx + ! use the EOS with constant entropy to find corrected state + eos_state % p = p_want + eos_state % s = entropy_fixed + eos_state % xn(:) = model_params % xn(:) - ! entropy never changes in this model - s = entropy_fixed + call eos(eos_input_ps, eos_state) - k1 = f(p, s, const_grav, rho, T, xn) - k2 = f(p + HALF*h*k1, s, const_grav, rho, T, xn) - k3 = f(p + HALF*h*k2, s, const_grav, rho, T, xn) - k4 = f(p + h*k3, s, const_grav, rho, T, xn) + drho = eos_state % rho - dens_zone + dens_zone = eos_state % rho + if (abs(drho) < TOL*dens_zone) then + converged_hse = .TRUE. + exit + end if - pnew = p + SIXTH*h*(k1 + TWO*k2 + TWO*k3 + k4) + end do - ! call the EOS to get the remainder of the thermodynamics - eos_state % T = T ! initial guess - eos_state % rho = rho ! initial guess - eos_state % xn(:) = model_params % xn(:) - eos_state % p = pnew - eos_state % s = s + if (.not. converged_hse) then + print *, "failed to convergence in initial model generation" + call castro_error("ERROR") + end if - call eos(eos_input_ps, eos_state) + ! initialze zone + model_state(i, idens_model) = dens_zone + model_state(i, ipres_model) = p_want + model_state(i, itemp_model) = eos_state % T + model_state(i, ispec_model:ispec_model-1+nspec) = xn(:) - ! update the thermodynamics in this zone - model_state(i, idens_model) = eos_state % rho - model_state(i, itemp_model) = eos_state % T - model_state(i, ipres_model) = eos_state % p - model_state(i, ispec_model:ispec_model-1+nspec) = eos_state % xn(:) + end if + print *, i, model_state(i, idens_model), model_state(i, ipres_model) - ! reset for the next iteration - p = pnew + end do - enddo + end if end subroutine generate_initial_model From edea55fd18f2f543828453073736b5e6e607b877 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 16:41:08 -0400 Subject: [PATCH 17/72] add well balanced test --- .../bubble_convergence/inputs_1d.well_balance | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance diff --git a/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance new file mode 100644 index 0000000000..d0375785ac --- /dev/null +++ b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance @@ -0,0 +1,79 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000 +stop_time = 0.1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 0 0 +geometry.coord_sys = 0 # 0 => cart, 1 => RZ 2=>spherical +geometry.prob_lo = 0 0 +geometry.prob_hi = 7.68e6 7.68e6 +amr.n_cell = 256 256 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 1 3 +castro.hi_bc = 1 3 + +castro.xl_ext_bc_type = "hse" +castro.xr_ext_bc_type = "hse" + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 1 +castro.do_grav = 1 +castro.do_sponge = 1 + +castro.small_temp = 1.e6 + +castro.sdc_order=2 +castro.ppm_type=0 +castro.plm_well_balanced=1 +castro.time_integration_method=2 +castro.limit_fourth_order=1 +castro.use_reconstructed_gamma1=1 +castro.sdc_solve_for_rhoe=1 +castro.sdc_solver_tol_spec=1.e-10 +castro.sdc_solver_tol_dens=1.e-10 +castro.sdc_solver_tol_ener=1.e-5 +castro.sdc_solver_atol=1.e-10 +castro.sdc_solver=1 + +castro.grav_source_type = 2 + +gravity.gravity_type = ConstantGrav +gravity.const_grav = -1.e10 + +# TIME STEP CONTROL +castro.cfl = 0.8 # cfl number for hyperbolic system +castro.init_shrink = 1.0 # scale back initial timestep +castro.change_max = 1.0 # max time step growth +castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.fixed_dt = 0.375e-4 + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +amr.ref_ratio = 2 2 2 2 # refinement ratio +amr.regrid_int = 2 2 2 2 # how often to regrid +amr.blocking_factor = 4 # block factor in grid generation +amr.max_grid_size = 64 +amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est + +# CHECKPOINT FILES +amr.check_file = chk # root name of checkpoint file +amr.check_int = 1000 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = bubble_256_plt # root name of plotfile +amr.plot_per = 1e-3 # number of timesteps between plotfiles +amr.derive_plot_vars = ALL + +#PROBIN FILENAME +amr.probin_file = probin.256.nopert From c49786a37d343c11dcabbdab8a76aeb0754bce7f Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 23 Aug 2019 16:54:53 -0400 Subject: [PATCH 18/72] fix the 2nd order model the well-balanced scheme now works --- .../bubble_convergence/initial_model.F90 | 45 ++++++++++++++++++- .../bubble_convergence/inputs_1d.well_balance | 2 +- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Exec/reacting_tests/bubble_convergence/initial_model.F90 b/Exec/reacting_tests/bubble_convergence/initial_model.F90 index 4cd78d603a..285705d439 100644 --- a/Exec/reacting_tests/bubble_convergence/initial_model.F90 +++ b/Exec/reacting_tests/bubble_convergence/initial_model.F90 @@ -205,7 +205,7 @@ subroutine generate_initial_model(nx, xmin, xmax, model_params, nbuf) model_state(i, idens_model) = eos_state % rho model_state(i, itemp_model) = eos_state % T - model_state(i, ispec_model:ispec_model-1+nspec) = xn(:) + model_state(i, ispec_model:ispec_model-1+nspec) = model_params % xn(:) else @@ -243,13 +243,54 @@ subroutine generate_initial_model(nx, xmin, xmax, model_params, nbuf) model_state(i, idens_model) = dens_zone model_state(i, ipres_model) = p_want model_state(i, itemp_model) = eos_state % T - model_state(i, ispec_model:ispec_model-1+nspec) = xn(:) + model_state(i, ispec_model:ispec_model-1+nspec) = model_params % xn(:) end if print *, i, model_state(i, idens_model), model_state(i, ipres_model) end do + ! now integrate down + do i = ibase-1, 1, -1 + + dens_zone = model_state(i+1, idens_model) + + converged_hse = .FALSE. + + do iter = 1, MAX_ITER + + p_want = model_state(i+1, ipres_model) - & + dx*HALF*(dens_zone + model_state(i+1, idens_model))*const_grav + + ! use the EOS with constant entropy to find corrected state + eos_state % p = p_want + eos_state % s = entropy_fixed + eos_state % xn(:) = model_params % xn(:) + + call eos(eos_input_ps, eos_state) + + drho = eos_state % rho - dens_zone + dens_zone = eos_state % rho + if (abs(drho) < TOL*dens_zone) then + converged_hse = .TRUE. + exit + end if + + end do + + if (.not. converged_hse) then + print *, "failed to convergence in initial model generation" + call castro_error("ERROR") + end if + + ! initialze zone + model_state(i, idens_model) = dens_zone + model_state(i, ipres_model) = p_want + model_state(i, itemp_model) = eos_state % T + model_state(i, ispec_model:ispec_model-1+nspec) = model_params % xn(:) + + enddo + end if end subroutine generate_initial_model diff --git a/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance index d0375785ac..ec6e336f57 100644 --- a/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance +++ b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance @@ -22,7 +22,7 @@ castro.xr_ext_bc_type = "hse" # WHICH PHYSICS castro.do_hydro = 1 -castro.do_react = 1 +castro.do_react = 0 castro.do_grav = 1 castro.do_sponge = 1 From 97fea33adce402c01aabc4c1bcd982e8d8852c74 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sat, 24 Aug 2019 12:06:11 -0400 Subject: [PATCH 19/72] set plm_iorder --- Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance | 1 + 1 file changed, 1 insertion(+) diff --git a/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance index ec6e336f57..a09f6ba57a 100644 --- a/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance +++ b/Exec/reacting_tests/bubble_convergence/inputs_1d.well_balance @@ -30,6 +30,7 @@ castro.small_temp = 1.e6 castro.sdc_order=2 castro.ppm_type=0 +castro.plm_iorder = 2 castro.plm_well_balanced=1 castro.time_integration_method=2 castro.limit_fourth_order=1 From 880a29528f322f4a548f7ef726cb1698e9c3ae1a Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sat, 24 Aug 2019 17:38:36 -0400 Subject: [PATCH 20/72] a simple plotting routine --- .../bubble_convergence/analysis/hse.gp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 Exec/reacting_tests/bubble_convergence/analysis/hse.gp diff --git a/Exec/reacting_tests/bubble_convergence/analysis/hse.gp b/Exec/reacting_tests/bubble_convergence/analysis/hse.gp new file mode 100644 index 0000000000..53939c06d9 --- /dev/null +++ b/Exec/reacting_tests/bubble_convergence/analysis/hse.gp @@ -0,0 +1,27 @@ +# plot the Sod results. Here we assume that we have a file called sedov.out +# with columns containing x, rho, xmom, pres + +set term pngcairo size 800, 1000 +set output 'hse.png' + +set multiplot; + +set size 1, 0.33; + +#set xrange [0:0.5] +set xlabel "x"; + +set origin 0.0, 0.666; +set ylabel "density"; +plot 'bubble_256_plt02667.slice' using 1:2 notitle w l + +set origin 0.0, 0.333; +set ylabel "velocity"; +plot 'bubble_256_plt02667.slice' using 1:4 notitle w l + +set origin 0.0, 0.0; +set ylabel "temperature"; +plot 'bubble_256_plt02667.slice' using 1:3 notitle w l + +unset multiplot; +set term x11; From 63c56f4d64346e7de68c1d5f6aedb4a39b2b22c8 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sat, 24 Aug 2019 19:16:25 -0400 Subject: [PATCH 21/72] remove unneeded parameter --- Source/hydro/Castro_mol_nd.F90 | 6 +++--- Source/hydro/slope_nd.F90 | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index c3d7282715..de6ba93939 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -18,7 +18,7 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & use castro_error_module use meth_params_module, only : NQ, NVAR, NGDNV, GDPRES, & UTEMP, UMX, & - use_flattening, plm_well_balanced, QPRES, & + plm_well_balanced, QPRES, & QTEMP, QFS, QFX, QREINT, QRHO, QU, QV, QW, & first_order_hydro, hybrid_riemann, & ppm_temp_fix, const_grav @@ -298,7 +298,7 @@ subroutine ca_mol_ppm_reconstruct(lo, hi, & use castro_error_module use meth_params_module, only : NQ, NVAR, NGDNV, GDPRES, & UTEMP, UMX, & - use_flattening, QPRES, & + QPRES, & QTEMP, QFS, QFX, QREINT, QRHO, & first_order_hydro, hybrid_riemann, & ppm_temp_fix @@ -448,7 +448,7 @@ subroutine ca_mol_consup(lo, hi, & use castro_error_module use meth_params_module, only : NQ, NVAR, NGDNV, GDPRES, & UTEMP, UMX, & - use_flattening, QPRES, & + QPRES, & QTEMP, QFS, QFX, QREINT, QRHO, & first_order_hydro, difmag, hybrid_riemann, & limit_fluxes_on_small_dens, ppm_type, ppm_temp_fix diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index c1354ebf10..956f0f2947 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -95,6 +95,7 @@ subroutine uslope(lo, hi, idir, & dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) + else if (plm_iorder == 2) then ! the 2nd order MC limiter From 3866e0b0c8dcb7771ab6b8cee6b0ce6e67f4bc6f Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 25 Aug 2019 14:17:21 -0700 Subject: [PATCH 22/72] a uniform grid SDC inputs --- .../science/flame_wave/inputs.boost_10_10.sdc | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Exec/science/flame_wave/inputs.boost_10_10.sdc b/Exec/science/flame_wave/inputs.boost_10_10.sdc index fb880d2960..b9244ae287 100644 --- a/Exec/science/flame_wave/inputs.boost_10_10.sdc +++ b/Exec/science/flame_wave/inputs.boost_10_10.sdc @@ -7,19 +7,19 @@ geometry.is_periodic = 0 0 geometry.coord_sys = 1 # 0 => cart, 1 => RZ 2=>spherical geometry.prob_lo = 0 0 geometry.prob_hi = 1.2288e5 3.072e4 -amr.n_cell = 768 192 +amr.n_cell = 6144 1536 # >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< # 0 = Interior 3 = Symmetry # 1 = Inflow 4 = SlipWall # 2 = Outflow 5 = NoSlipWall # >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< -castro.lo_bc = 3 3 +castro.lo_bc = 3 1 castro.hi_bc = 2 1 castro.yl_ext_bc_type = "hse" castro.hse_interp_temp = 1 -castro.hse_reflect_vels = 1 +castro.hse_reflect_vels = 0 castro.yr_ext_bc_type = "interp" @@ -35,10 +35,15 @@ castro.small_dens = 1.e-5 castro.time_integration_method = 2 castro.sdc_order = 2 +castro.ppm_type = 0 +castro.plm_iorder = 2 +castro.plm_well_balanced = 1 + castro.sdc_solve_for_rhoe = 1 castro.sdc_solver_tol_dens = 1.e-10 castro.sdc_solver_tol_spec = 1.e-10 -castro.sdc_solver_tol_ener = 1.e-10 +castro.sdc_solver_tol_ener = 1.e-6 +castro.sdc_solver_atol = 1.e-10 castro.sdc_solver = 1 castro.grav_source_type = 2 @@ -51,21 +56,22 @@ castro.rotation_include_centrifugal = 0 castro.diffuse_temp = 1 castro.diffuse_cutoff_density_hi = 5.e4 -castro.diffuse_cutoff_density = 2.e4 +#castro.diffuse_cutoff_density = 2.e4 castro.diffuse_cond_scale_fac = 10.0 -castro.react_rho_min = 1.e2 +castro.react_rho_min = 1.e3 castro.react_rho_max = 1.5e7 castro.react_T_min = 6.e7 # TIME STEP CONTROL -castro.cfl = 0.8 # cfl number for hyperbolic system +castro.cfl = 0.5 # cfl number for hyperbolic system castro.init_shrink = 0.1 # scale back initial timestep castro.change_max = 1.1 # max time step growth castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.dtnuc_e = 0.1 # DIAGNOSTICS & VERBOSITY castro.sum_interval = 1 # timesteps between computing mass @@ -73,7 +79,7 @@ castro.v = 1 # verbosity in Castro.cpp amr.v = 1 # verbosity in Amr.cpp # REFINEMENT / REGRIDDING -amr.max_level = 2 # maximum level number allowed +amr.max_level = 0 # maximum level number allowed amr.ref_ratio = 4 2 2 2 # refinement ratio amr.regrid_int = 2 2 2 2 # how often to regrid amr.blocking_factor = 16 # block factor in grid generation @@ -90,7 +96,8 @@ amr.plot_per = 2.e-3 # number of seconds between plotfiles amr.derive_plot_vars = ALL amr.small_plot_file = flame_wave_smallplt # root name of plotfile -amr.small_plot_per = 5.e-5 # number of seconds between plotfiles +#amr.small_plot_per = 5.e-5 # number of seconds between plotfiles +amr.plot_int = 10 amr.small_plot_vars = density Temp enuc amr.derive_small_plot_vars = abar x_velocity y_velocity z_velocity From 37051cc7a06abd741e8276ea142e7394b8dd0d44 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 26 Aug 2019 09:01:56 -0400 Subject: [PATCH 23/72] implement well-balancing for the other directions --- Source/hydro/Castro_mol_nd.F90 | 75 ++++++++-- Source/hydro/slope_nd.F90 | 257 +++++++++++++++++++++++---------- 2 files changed, 242 insertions(+), 90 deletions(-) diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index de6ba93939..04d532236e 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -102,39 +102,82 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & end do end do end if + #if BL_SPACEDIM >= 2 else if (idir == 2) then - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - ! left state at j+1/2 interface - qm(i,j+1,k,n,2) = q(i,j,k,n) + HALF*dq(i,j,k,n) + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) - ! right state at j-1/2 interface - qp(i,j,k,n,2) = q(i,j,k,n) - HALF*dq(i,j,k,n) + ! left state at i+1/2 interface + qm(i,j+1,k,n,2) = q(i,j,k,n) + HALF*dq(i,j,k,n) + & + HALF*dx(2)*q(i,j,k,QRHO)*const_grav + ! right state at i-1/2 interface + qp(i,j,k,n,2) = q(i,j,k,n) - HALF*dq(i,j,k,n) - & + HALF*dx(2)*q(i,j,k,QRHO)*const_grav + end do end do end do - end do + + else + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + ! left state at j+1/2 interface + qm(i,j+1,k,n,2) = q(i,j,k,n) + HALF*dq(i,j,k,n) + + ! right state at j-1/2 interface + qp(i,j,k,n,2) = q(i,j,k,n) - HALF*dq(i,j,k,n) + + end do + end do + end do + end if #endif #if BL_SPACEDIM == 3 else - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then - ! left state at k+1/2 interface - qm(i,j,k+1,n,3) = q(i,j,k,n) + HALF*dq(i,j,k,n) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) - ! right state at k-1/2 interface - qp(i,j,k,n,3) = q(i,j,k,n) - HALF*dq(i,j,k,n) + ! left state at i+1/2 interface + qm(i,j,k+1,n,3) = q(i,j,k,n) + HALF*dq(i,j,k,n) + & + HALF*dx(3)*q(i,j,k,QRHO)*const_grav + ! right state at i-1/2 interface + qp(i,j,k,n,3) = q(i,j,k,n) - HALF*dq(i,j,k,n) - & + HALF*dx(3)*q(i,j,k,QRHO)*const_grav + end do end do end do - end do + + else + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + ! left state at k+1/2 interface + qm(i,j,k+1,n,3) = q(i,j,k,n) + HALF*dq(i,j,k,n) + + ! right state at k-1/2 interface + qp(i,j,k,n,3) = q(i,j,k,n) - HALF*dq(i,j,k,n) + + end do + end do + end do + + end if #endif end if diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index 956f0f2947..829a999b14 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -195,49 +195,102 @@ subroutine uslope(lo, hi, idir, & do j = lo(2), hi(2) do i = lo(1), hi(1) - ! First compute Fromm slopes + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then + ! we'll only do a second-order pressure slope, + ! but we'll follow the well-balanced scheme of + ! Kappeli. Note at the moment we are assuming + ! constant gravity. + p0 = ZERO + pp1 = q(i,j+1,k,QPRES) - (p0 + HALF*dx(2)*(q(i,j,k,QRHO) + q(i,j+1,k,QRHO))*const_grav) + pm1 = q(i,j+1,k,QPRES) - (p0 - HALF*dx(2)*(q(i,j,k,QRHO) + q(i,j-1,k,QRHO))*const_grav) - ! df at j+1 - dlftp1 = TWO*(q(i,j+1,k,n) - q(i,j,k,n)) - drgtp1 = TWO*(q(i,j+2,k,n) - q(i,j+1,k,n)) - dcen = FOURTH * (dlftp1 + drgtp1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftp1), abs(drgtp1)) - if (dlftp1*drgtp1 >= ZERO) then - dlim = slop - else - dlim = ZERO - end if - dfp1 = dsgn*min(dlim, abs(dcen)) + if (j == domlo(2) .and. physbc_lo(2) == Symmetry) then + pm1 = ZERO ! HSE is perfectly satisfied + end if - ! df at j-1 - dlftm1 = TWO*(q(i,j-1,k,n) - q(i,j-2,k,n)) - drgtm1 = TWO*(q(i,j,k,n) - q(i,j-1,k,n)) - dcen = FOURTH * (dlftm1 + drgtm1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftm1), abs(drgtm1)) - if (dlftm1*drgtm1 >= ZERO) then - dlim = slop - else - dlim = ZERO - end if - dfm1 = dsgn*min(dlim, abs(dcen)) + if (j == domhi(2) .and. physbc_hi(2) == Symmetry) then + pp1 = ZERO + end if + + dlft = TWO*(p0 - pm1) + drgt = TWO*(pp1 - p0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) + + + else if (plm_iorder == 2) then + ! the 2nd order MC limiter + + qm1 = q(i,j-1,k,n) + q0 = q(i,j,k,n) + qp1 = q(i,j+1,k,n) + + dlft = TWO*(q0 - qm1) + drgt = TWO*(qp1 - q0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - ! Now compute limited fourth order slopes at j - dlft = drgtm1 - drgt = dlftp1 - dcen = FOURTH * (dlft + drgt) - dsgn = sign(ONE, dcen) - slop = min(abs(dlft), abs(drgt)) - if (dlft*drgt >= ZERO) then - dlim = slop else - dlim = ZERO - end if + ! First compute Fromm slopes - dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) - dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) + ! df at j+1 + dlftp1 = TWO*(q(i,j+1,k,n) - q(i,j,k,n)) + drgtp1 = TWO*(q(i,j+2,k,n) - q(i,j+1,k,n)) + dcen = FOURTH * (dlftp1 + drgtp1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftp1), abs(drgtp1)) + if (dlftp1*drgtp1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfp1 = dsgn*min(dlim, abs(dcen)) + ! df at j-1 + dlftm1 = TWO*(q(i,j-1,k,n) - q(i,j-2,k,n)) + drgtm1 = TWO*(q(i,j,k,n) - q(i,j-1,k,n)) + dcen = FOURTH * (dlftm1 + drgtm1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftm1), abs(drgtm1)) + if (dlftm1*drgtm1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfm1 = dsgn*min(dlim, abs(dcen)) + + ! Now compute limited fourth order slopes at j + dlft = drgtm1 + drgt = dlftp1 + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) + + end if end do end do end do @@ -251,48 +304,104 @@ subroutine uslope(lo, hi, idir, & do j = lo(2), hi(2) do i = lo(1), hi(1) - ! First compute Fromm slopes + if (plm_well_balanced == 1 .and. n == QPRES .and. idir == AMREX_SPACEDIM) then + ! we'll only do a second-order pressure slope, + ! but we'll follow the well-balanced scheme of + ! Kappeli. Note at the moment we are assuming + ! constant gravity. + p0 = ZERO + pp1 = q(i,j,k+1,QPRES) - (p0 + HALF*dx(3)*(q(i,j,k,QRHO) + q(i,j,k+1,QRHO))*const_grav) + pm1 = q(i,j,k-1,QPRES) - (p0 - HALF*dx(3)*(q(i,j,k,QRHO) + q(i,j,k-1,QRHO))*const_grav) - ! df at k+1 - dlftp1 = TWO*(q(i,j,k+1,n) - q(i,j,k,n)) - drgtp1 = TWO*(q(i,j,k+2,n) - q(i,j,k+1,n)) - dcen = FOURTH * (dlftp1 + drgtp1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftp1), abs(drgtp1)) - if (dlftp1*drgtp1 >= ZERO) then - dlim = slop - else - dlim = ZERO - end if - dfp1 = dsgn*min(dlim, abs(dcen)) + if (k == domlo(3) .and. physbc_lo(3) == Symmetry) then + pm1 = ZERO ! HSE is perfectly satisfied + end if - ! df at k-1 - dlftm1 = TWO*(q(i,j,k-1,n) - q(i,j,k-2,n)) - drgtm1 = TWO*(q(i,j,k,n) - q(i,j,k-1,n)) - dcen = FOURTH * (dlftm1 + drgtm1) - dsgn = sign(ONE, dcen) - slop = min(abs(dlftm1), abs(drgtm1)) - if (dlftm1*drgtm1 >= ZERO) then - dlim = slop - else - dlim = ZERO - end if - dfm1 = dsgn*min(dlim, abs(dcen)) + if (k == domhi(3) .and. physbc_hi(3) == Symmetry) then + pp1 = ZERO + end if + + dlft = TWO*(p0 - pm1) + drgt = TWO*(pp1 - p0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) + + + else if (plm_iorder == 2) then + ! the 2nd order MC limiter + + qm1 = q(i,j,k-1,n) + q0 = q(i,j,k,n) + qp1 = q(i,j,k+1,n) + + dlft = TWO*(q0 - qm1) + drgt = TWO*(qp1 - q0) + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - ! Now compute limited fourth order slopes at k - dlft = drgtm1 - drgt = dlftp1 - dcen = FOURTH * (dlft + drgt) - dsgn = sign(ONE, dcen) - slop = min(abs(dlft), abs(drgt)) - if (dlft*drgt >= ZERO) then - dlim = slop else - dlim = ZERO - end if + ! the 4th order MC limiter - dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) - dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) + ! First compute Fromm slopes + + ! df at k+1 + dlftp1 = TWO*(q(i,j,k+1,n) - q(i,j,k,n)) + drgtp1 = TWO*(q(i,j,k+2,n) - q(i,j,k+1,n)) + dcen = FOURTH * (dlftp1 + drgtp1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftp1), abs(drgtp1)) + if (dlftp1*drgtp1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfp1 = dsgn*min(dlim, abs(dcen)) + + ! df at k-1 + dlftm1 = TWO*(q(i,j,k-1,n) - q(i,j,k-2,n)) + drgtm1 = TWO*(q(i,j,k,n) - q(i,j,k-1,n)) + dcen = FOURTH * (dlftm1 + drgtm1) + dsgn = sign(ONE, dcen) + slop = min(abs(dlftm1), abs(drgtm1)) + if (dlftm1*drgtm1 >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + dfm1 = dsgn*min(dlim, abs(dcen)) + + ! Now compute limited fourth order slopes at k + dlft = drgtm1 + drgt = dlftp1 + dcen = FOURTH * (dlft + drgt) + dsgn = sign(ONE, dcen) + slop = min(abs(dlft), abs(drgt)) + if (dlft*drgt >= ZERO) then + dlim = slop + else + dlim = ZERO + end if + + dq1 = FOUR3RD*dcen - SIXTH*(dfp1 + dfm1) + dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dq1)) + + end if end do end do end do From 6658bcb9a8bd7a23b7d513da1afdc8721a211475 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 26 Aug 2019 11:37:22 -0400 Subject: [PATCH 24/72] +Y HSE BC for testing purposes --- Source/problems/bc_ext_fill_nd.F90 | 164 ++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 1 deletion(-) diff --git a/Source/problems/bc_ext_fill_nd.F90 b/Source/problems/bc_ext_fill_nd.F90 index 6c8c0c2f37..60f8b0468f 100644 --- a/Source/problems/bc_ext_fill_nd.F90 +++ b/Source/problems/bc_ext_fill_nd.F90 @@ -739,10 +739,172 @@ subroutine ext_fill(lo, hi, adv, adv_lo, adv_hi, & if (bc(2,2,1) == EXT_DIR .and. hi(2) > domhi(2)) then if (yr_ext == EXT_HSE) then + + do k = lo(3), hi(3) + do i = lo(1), hi(1) + + ! we are integrating along a column at constant i. + ! Make sure that our starting state is well-defined + dens_below = adv(i,domhi(2),k,URHO) + + ! sometimes, we might be working in a corner + ! where the ghost cells above us have not yet + ! been initialized. In that case, take the info + ! from the initial model + if (dens_below == ZERO) then + y = problo(2) + delta(2)*(dble(domhi(2)) + HALF) + + call interpolate_sub(dens_below, y, idens_model) + call interpolate_sub(temp_below, y, itemp_model) + + do m = 1, nspec + call interpolate_sub(X_zone(m), y, ispec_model-1+m) + end do + + else + temp_below = adv(i,domhi(2),k,UTEMP) + X_zone(:) = adv(i,domhi(2),k,UFS:UFS-1+nspec)/dens_below + endif + + ! keep track of the density at the base of the domain + dens_base = dens_below + + ! get pressure in this zone (the initial below zone) + eos_state%rho = dens_below + eos_state%T = temp_below + eos_state%xn(:) = X_zone(:) + + call eos(eos_input_rt, eos_state) + + eint = eos_state%e + pres_below = eos_state%p + + ! integrate upward + jmin = domhi(2)+1 + jmax = adv_hi(2) +#ifdef AMREX_USE_CUDA + ! For CUDA, this should only be one thread doing the work: + ! we'll arbitrary choose the zone with index domlo(1) - 1. + if (hi(2) /= jmax) then + jmax = jmin - 1 + end if +#endif + do j = jmin, jmax + y = problo(2) + delta(2)*(dble(j) + HALF) + + ! HSE integration to get density, pressure + + ! initial guesses + dens_zone = dens_below + + ! temperature and species held constant in BCs + if (hse_interp_temp == 1) then + call interpolate_sub(temp_zone, y, itemp_model) + else + temp_zone = temp_below + endif + + converged_hse = .FALSE. + + do iter = 1, MAX_ITER + + ! pressure needed from HSE + p_want = pres_below + & + delta(2)*HALF*(dens_zone + dens_below)*const_grav + + ! pressure from EOS + eos_state%rho = dens_zone + eos_state%T = temp_zone + eos_state%xn(:) = X_zone(:) + + call eos(eos_input_rt, eos_state) + + pres_zone = eos_state%p + dpdr = eos_state%dpdr + eint = eos_state%e + + ! Newton-Raphson - we want to zero A = p_want - p(rho) + A = p_want - pres_zone + drho = A/(dpdr - HALF*delta(2)*const_grav) + + dens_zone = max(0.9_rt*dens_zone, & + min(dens_zone + drho, 1.1_rt*dens_zone)) + + ! convergence? + if (abs(drho) < TOL*dens_zone) then + converged_hse = .TRUE. + exit + endif + + enddo + #ifndef AMREX_USE_CUDA - call castro_error("ERROR: HSE boundaries not implemented for +Y") + if (.not. converged_hse) then + print *, "i, j, k, domhi(2): ", i, j, k, domhi(2) + print *, "p_want: ", p_want + print *, "dens_zone: ", dens_zone + print *, "temp_zone: ", temp_zone + print *, "drho: ", drho + print *, " " + print *, "column info: " + print *, " dens: ", adv(i,j:domhi(2),k,URHO) + print *, " temp: ", adv(i,j:domhi(2),k,UTEMP) + call castro_error("ERROR in bc_ext_fill_nd: failure to converge in +Y BC") + endif #endif + ! velocity + if (hse_zero_vels == 1) then + + ! zero normal momentum causes pi waves to pass through + adv(i,j,k,UMX) = ZERO + adv(i,j,k,UMY) = ZERO + adv(i,j,k,UMZ) = ZERO + + else + + if (hse_reflect_vels == 1) then + ! reflect normal, zero gradient for transverse + ! note: we need to match the corresponding + ! zone on the other side of the interface + joff = j-domhi(2)-1 + adv(i,j,k,UMY) = -dens_zone*(adv(i,domhi(2)-joff,k,UMY)/adv(i,domhi(2)-joff,k,URHO)) + + adv(i,j,k,UMX) = -dens_zone*(adv(i,domhi(2),k,UMX)/dens_base) + adv(i,j,k,UMZ) = -dens_zone*(adv(i,domhi(2),k,UMZ)/dens_base) + else + ! zero gradient + adv(i,j,k,UMX) = dens_zone*(adv(i,domhi(2),k,UMX)/dens_base) + adv(i,j,k,UMY) = dens_zone*(adv(i,domhi(2),k,UMY)/dens_base) + adv(i,j,k,UMZ) = dens_zone*(adv(i,domhi(2),k,UMZ)/dens_base) + endif + endif + eos_state%rho = dens_zone + eos_state%T = temp_zone + eos_state%xn(:) = X_zone + + call eos(eos_input_rt, eos_state) + + pres_zone = eos_state%p + eint = eos_state%e + + ! store the final state + adv(i,j,k,URHO) = dens_zone + adv(i,j,k,UEINT) = dens_zone*eint + adv(i,j,k,UEDEN) = dens_zone*eint + & + HALF*sum(adv(i,j,k,UMX:UMZ)**2)/dens_zone + adv(i,j,k,UTEMP) = temp_zone + adv(i,j,k,UFS:UFS-1+nspec) = dens_zone*X_zone(:) + + ! for the next zone + dens_below = dens_zone + pres_below = pres_zone + + end do + end do + end do + + elseif (yr_ext == EXT_INTERP) then ! interpolate thermodynamics from initial model From d2e010215404c2221b98309a254f61b5d7359a48 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 26 Aug 2019 11:39:17 -0400 Subject: [PATCH 25/72] a well-balanced test in 2-d --- .../bubble_convergence/inputs_2d.well_balance | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance diff --git a/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance b/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance new file mode 100644 index 0000000000..a7ac88cd15 --- /dev/null +++ b/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance @@ -0,0 +1,80 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000 +stop_time = 0.1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 0 0 +geometry.coord_sys = 0 # 0 => cart, 1 => RZ 2=>spherical +geometry.prob_lo = 0 0 +geometry.prob_hi = 7.68e6 7.68e6 +amr.n_cell = 256 256 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 3 1 +castro.hi_bc = 3 1 + +castro.yl_ext_bc_type = "hse" +castro.yr_ext_bc_type = "hse" + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 0 +castro.do_grav = 1 +castro.do_sponge = 1 + +castro.small_temp = 1.e6 + +castro.sdc_order=2 +castro.ppm_type=0 +castro.plm_iorder = 2 +castro.plm_well_balanced=1 +castro.time_integration_method=2 +castro.limit_fourth_order=1 +castro.use_reconstructed_gamma1=1 +castro.sdc_solve_for_rhoe=1 +castro.sdc_solver_tol_spec=1.e-10 +castro.sdc_solver_tol_dens=1.e-10 +castro.sdc_solver_tol_ener=1.e-5 +castro.sdc_solver_atol=1.e-10 +castro.sdc_solver=1 + +castro.grav_source_type = 2 + +gravity.gravity_type = ConstantGrav +gravity.const_grav = -1.e10 + +# TIME STEP CONTROL +castro.cfl = 0.8 # cfl number for hyperbolic system +castro.init_shrink = 1.0 # scale back initial timestep +castro.change_max = 1.0 # max time step growth +castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.fixed_dt = 0.375e-4 + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +amr.ref_ratio = 2 2 2 2 # refinement ratio +amr.regrid_int = 2 2 2 2 # how often to regrid +amr.blocking_factor = 4 # block factor in grid generation +amr.max_grid_size = 64 +amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est + +# CHECKPOINT FILES +amr.check_file = chk # root name of checkpoint file +amr.check_int = 1000 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = bubble_256_plt # root name of plotfile +amr.plot_per = 1e-3 # number of timesteps between plotfiles +amr.derive_plot_vars = ALL + +#PROBIN FILENAME +amr.probin_file = probin.256.nopert From 1c400f0fdb66f1c620eda30d04c24184fce0e0be Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 26 Aug 2019 11:49:51 -0400 Subject: [PATCH 26/72] switch the 2-d well-balanced test to r-z --- Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance b/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance index a7ac88cd15..9e99892a66 100644 --- a/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance +++ b/Exec/reacting_tests/bubble_convergence/inputs_2d.well_balance @@ -4,7 +4,7 @@ stop_time = 0.1 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 0 0 -geometry.coord_sys = 0 # 0 => cart, 1 => RZ 2=>spherical +geometry.coord_sys = 1 # 0 => cart, 1 => RZ 2=>spherical geometry.prob_lo = 0 0 geometry.prob_hi = 7.68e6 7.68e6 amr.n_cell = 256 256 @@ -15,7 +15,7 @@ amr.n_cell = 256 256 # 2 = Outflow 5 = NoSlipWall # >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< castro.lo_bc = 3 1 -castro.hi_bc = 3 1 +castro.hi_bc = 2 1 castro.yl_ext_bc_type = "hse" castro.yr_ext_bc_type = "hse" From 28c1d49cd15e15c2fa672f330f83f4ba33e4d0ed Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 27 Aug 2019 14:22:32 -0400 Subject: [PATCH 27/72] a small uniform grid test --- .../flame_wave/inputs.boost_10_10.sdc.small | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 Exec/science/flame_wave/inputs.boost_10_10.sdc.small diff --git a/Exec/science/flame_wave/inputs.boost_10_10.sdc.small b/Exec/science/flame_wave/inputs.boost_10_10.sdc.small new file mode 100644 index 0000000000..b55ab09900 --- /dev/null +++ b/Exec/science/flame_wave/inputs.boost_10_10.sdc.small @@ -0,0 +1,105 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 9900000 +stop_time = 3.0 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 0 0 +geometry.coord_sys = 1 # 0 => cart, 1 => RZ 2=>spherical +geometry.prob_lo = 0 0 +geometry.prob_hi = 4.096e4 2.048e4 +amr.n_cell = 2048 1024 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 3 1 +castro.hi_bc = 2 1 + +castro.yl_ext_bc_type = "hse" +castro.hse_interp_temp = 1 +castro.hse_reflect_vels = 0 + +castro.yr_ext_bc_type = "interp" + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 1 +castro.do_rotation = 1 +castro.do_grav = 1 +castro.do_sponge = 1 + +castro.small_temp = 1.e6 +castro.small_dens = 1.e-5 + +castro.time_integration_method = 2 +castro.sdc_order = 2 +castro.ppm_type = 1 +castro.plm_iorder = 2 +castro.plm_well_balanced = 0 + +castro.sdc_solve_for_rhoe = 1 +castro.sdc_solver_tol_dens = 1.e-10 +castro.sdc_solver_tol_spec = 1.e-10 +castro.sdc_solver_tol_ener = 1.e-6 +castro.sdc_solver_atol = 1.e-10 +castro.sdc_solver = 1 + +castro.grav_source_type = 2 + +gravity.gravity_type = ConstantGrav +gravity.const_grav = -1.5e14 + +castro.rotational_period = 0.0005 +castro.rotation_include_centrifugal = 0 + +castro.diffuse_temp = 1 +castro.diffuse_cutoff_density_hi = 5.e4 +castro.diffuse_cutoff_density = 4.e4 + +castro.diffuse_cond_scale_fac = 10.0 + +castro.react_rho_min = 1.e3 +castro.react_rho_max = 1.5e7 + +castro.react_T_min = 6.e7 + + +# TIME STEP CONTROL +castro.cfl = 0.75 # cfl number for hyperbolic system +castro.init_shrink = 0.1 # scale back initial timestep +castro.change_max = 1.1 # max time step growth +castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.dtnuc_e = 0.25 + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +amr.ref_ratio = 4 2 2 2 # refinement ratio +amr.regrid_int = 2 2 2 2 # how often to regrid +amr.blocking_factor = 16 # block factor in grid generation +amr.max_grid_size = 128 +amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est + +# CHECKPOINT FILES +amr.check_file = flame_wave_chk # root name of checkpoint file +amr.check_int = 100 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = flame_wave_plt # root name of plotfile +amr.plot_per = 2.e-3 # number of seconds between plotfiles +amr.derive_plot_vars = ALL + +amr.small_plot_file = flame_wave_smallplt # root name of plotfile +#amr.small_plot_per = 5.e-5 # number of seconds between plotfiles +amr.plot_int = 10 +amr.small_plot_vars = density Temp enuc +amr.derive_small_plot_vars = abar x_velocity y_velocity z_velocity + +#PROBIN FILENAME +amr.probin_file = probin.boost_10_10 From 802448abedc73b0da7ecefad49079f2ee297ab05 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 28 Aug 2019 12:39:54 -0400 Subject: [PATCH 28/72] update --- Exec/science/flame_wave/inputs.boost_10_10.sdc.small | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Exec/science/flame_wave/inputs.boost_10_10.sdc.small b/Exec/science/flame_wave/inputs.boost_10_10.sdc.small index b55ab09900..3c91ba7209 100644 --- a/Exec/science/flame_wave/inputs.boost_10_10.sdc.small +++ b/Exec/science/flame_wave/inputs.boost_10_10.sdc.small @@ -19,7 +19,7 @@ castro.hi_bc = 2 1 castro.yl_ext_bc_type = "hse" castro.hse_interp_temp = 1 -castro.hse_reflect_vels = 0 +castro.hse_reflect_vels = 1 castro.yr_ext_bc_type = "interp" @@ -102,4 +102,4 @@ amr.small_plot_vars = density Temp enuc amr.derive_small_plot_vars = abar x_velocity y_velocity z_velocity #PROBIN FILENAME -amr.probin_file = probin.boost_10_10 +amr.probin_file = probin.boost_10_10.sdc From 5024913737d278bf8422549da16807d216c08682 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 28 Aug 2019 12:40:26 -0400 Subject: [PATCH 29/72] sdc probin --- .../science/flame_wave/probin.boost_10_10.sdc | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 Exec/science/flame_wave/probin.boost_10_10.sdc diff --git a/Exec/science/flame_wave/probin.boost_10_10.sdc b/Exec/science/flame_wave/probin.boost_10_10.sdc new file mode 100644 index 0000000000..b306dcc67f --- /dev/null +++ b/Exec/science/flame_wave/probin.boost_10_10.sdc @@ -0,0 +1,51 @@ +&fortin + + dtemp = 1.2e9 + x_half_max = 2.048d4 + x_half_width = 2048.d0 + + dx_model = 10.d0 + + dens_base = 3.43e6 + + T_star = 1.d8 + T_hi = 2.d8 + T_lo = 5.d6 + + H_star = 2000.d0 + atm_delta = 50.0 + + fuel1_name = "helium-4" + fuel1_frac = 1.0d0 + + ash1_name = "nickel-56" + ash1_frac = 1.0d0 + + low_density_cutoff = 1.d-4 + + cutoff_density = 2.5e4 + max_hse_tagging_level = 3 + max_base_tagging_level = 2 + + X_min = 1.e-2 + + x_refine_distance = 6.144e4 +/ + +&sponge + + sponge_upper_density = 1.d2 + sponge_lower_density = 1.d0 + sponge_timescale = 1.d-7 + +/ + + +&extern + + rtol_spec = 1.d-6 + atol_spec = 1.d-6 + + react_boost = 10.0 + use_tables = T +/ \ No newline at end of file From e8cbf85966b49a88752bf5138a1588f0dfdc40c0 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 2 Sep 2019 06:44:54 -0400 Subject: [PATCH 30/72] update changes --- CHANGES | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES b/CHANGES index 813dc483ba..801a14869e 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,18 @@ recent changes -- The SDC/MOL PLM reconstruction now implements reflecting BCs on the interface states + -- A well-balanced scheme has been added to the piecewise linear SDC + method, enabled with castro.plm_well_balanaced=1. At the moment + it only supports constant gravity. (#294) + + -- The weighting of the time-node fluxes stored in the flux registers + for SDC has been fixed (#658) + + -- The meaning of the castro.plm_iorder flag has changed slightly. + 1 = piecewise constant, 2 = 2nd order MC limiter, and >= 3 for + 4th order MC limiter (previously there was no way to select the + 2nd order limiter). + # 19.08.1 -- Fix CUDA compilation From fb320ea83eabc40f683151b93d9322d4cfa54106 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 2 Sep 2019 08:48:56 -0400 Subject: [PATCH 31/72] return to capture change in plm_iorder default --- Source/driver/meth_params.F90 | 2 +- Source/driver/param_includes/castro_defaults.H | 2 +- Source/driver/param_includes/castro_job_info_tests.H | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/driver/meth_params.F90 b/Source/driver/meth_params.F90 index 4f5c74782a..6f8dbc416a 100644 --- a/Source/driver/meth_params.F90 +++ b/Source/driver/meth_params.F90 @@ -560,7 +560,7 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para allocate(ppm_reference_eigenvectors) ppm_reference_eigenvectors = 0; allocate(plm_iorder) - plm_iorder = 2; + plm_iorder = 3; allocate(plm_well_balanced) plm_well_balanced = 0; allocate(hybrid_riemann) diff --git a/Source/driver/param_includes/castro_defaults.H b/Source/driver/param_includes/castro_defaults.H index 520bf9975c..0c037a1fc7 100644 --- a/Source/driver/param_includes/castro_defaults.H +++ b/Source/driver/param_includes/castro_defaults.H @@ -45,7 +45,7 @@ int Castro::ppm_type = 1; int Castro::ppm_temp_fix = 0; int Castro::ppm_predict_gammae = 0; int Castro::ppm_reference_eigenvectors = 0; -int Castro::plm_iorder = 2; +int Castro::plm_iorder = 3; int Castro::plm_well_balanced = 0; int Castro::hybrid_riemann = 0; int Castro::riemann_solver = 0; diff --git a/Source/driver/param_includes/castro_job_info_tests.H b/Source/driver/param_includes/castro_job_info_tests.H index 0d1978eab0..656ab647c4 100644 --- a/Source/driver/param_includes/castro_job_info_tests.H +++ b/Source/driver/param_includes/castro_job_info_tests.H @@ -40,7 +40,7 @@ jobInfoFile << (Castro::ppm_type == 1 ? " " : "[*] ") << "castro.ppm_type = " jobInfoFile << (Castro::ppm_temp_fix == 0 ? " " : "[*] ") << "castro.ppm_temp_fix = " << Castro::ppm_temp_fix << std::endl; jobInfoFile << (Castro::ppm_predict_gammae == 0 ? " " : "[*] ") << "castro.ppm_predict_gammae = " << Castro::ppm_predict_gammae << std::endl; jobInfoFile << (Castro::ppm_reference_eigenvectors == 0 ? " " : "[*] ") << "castro.ppm_reference_eigenvectors = " << Castro::ppm_reference_eigenvectors << std::endl; -jobInfoFile << (Castro::plm_iorder == 2 ? " " : "[*] ") << "castro.plm_iorder = " << Castro::plm_iorder << std::endl; +jobInfoFile << (Castro::plm_iorder == 3 ? " " : "[*] ") << "castro.plm_iorder = " << Castro::plm_iorder << std::endl; jobInfoFile << (Castro::plm_well_balanced == 0 ? " " : "[*] ") << "castro.plm_well_balanced = " << Castro::plm_well_balanced << std::endl; jobInfoFile << (Castro::hybrid_riemann == 0 ? " " : "[*] ") << "castro.hybrid_riemann = " << Castro::hybrid_riemann << std::endl; jobInfoFile << (Castro::riemann_solver == 0 ? " " : "[*] ") << "castro.riemann_solver = " << Castro::riemann_solver << std::endl; From 88637c90466b54b5dd5b0a7becfd1ca78be5782a Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 2 Sep 2019 09:00:45 -0400 Subject: [PATCH 32/72] fix compilation with old SDC --- Source/reactions/React_nd.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/reactions/React_nd.F90 b/Source/reactions/React_nd.F90 index 4f81389e28..ddebe45acf 100644 --- a/Source/reactions/React_nd.F90 +++ b/Source/reactions/React_nd.F90 @@ -230,6 +230,7 @@ subroutine ca_react_state_simplified_sdc(lo,hi, & use sdc_type_module, only : sdc_t, SRHO, SMX, SMZ, SEDEN, SEINT, SFS #endif use amrex_fort_module, only : rt => amrex_real + use react_util_module implicit none @@ -272,7 +273,7 @@ subroutine ca_react_state_simplified_sdc(lo,hi, & ! Don't burn if we're outside of the relevant (rho, T) range. - if (.not. okay_to_burn(U(i,j,k,:))) cycle + if (.not. okay_to_burn(uold(i,j,k,:))) cycle ! Feed in the old-time state data. From 34b259cf13887724f02c581405f4674ec9f99c19 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 3 Sep 2019 06:41:42 -0700 Subject: [PATCH 33/72] update changes --- CHANGES | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGES b/CHANGES index f7639ec5fd..d4e39d3598 100644 --- a/CHANGES +++ b/CHANGES @@ -1,13 +1,4 @@ -# 19.09 - - -- You can now type ./Castro.gnu.ex --describe to see the list of - modules / compilers the code was built with (#660) - - -- The reaction quantities are now computed as proper 4th order - averages for the plotfile, when using sdc_order = 4 (#647) - - -- The velerr tagging now takes the abs() of the velocity component - to ensure we tag large positive and negative velocities. +changes since the last release -- The SDC algorithm now implements the burning conditionals depending on rho and T (react_rho_min, react_rho_max, @@ -28,6 +19,18 @@ 4th order MC limiter (previously there was no way to select the 2nd order limiter). +# 19.09 + + -- You can now type ./Castro.gnu.ex --describe to see the list of + modules / compilers the code was built with (#660) + + -- The reaction quantities are now computed as proper 4th order + averages for the plotfile, when using sdc_order = 4 (#647) + + -- The velerr tagging now takes the abs() of the velocity component + to ensure we tag large positive and negative velocities. + + # 19.08.1 -- Fix CUDA compilation From 9fc51ebf1e9f95c8c33eee85b435c124987716e5 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 4 Sep 2019 04:34:39 -0400 Subject: [PATCH 34/72] new sdc_quadature parameter --- Source/driver/_cpp_parameters | 3 +++ Source/driver/param_includes/castro_defaults.H | 1 + Source/driver/param_includes/castro_job_info_tests.H | 1 + Source/driver/param_includes/castro_params.H | 1 + Source/driver/param_includes/castro_queries.H | 1 + 5 files changed, 7 insertions(+) diff --git a/Source/driver/_cpp_parameters b/Source/driver/_cpp_parameters index c64a3d8b4d..ae5db08422 100644 --- a/Source/driver/_cpp_parameters +++ b/Source/driver/_cpp_parameters @@ -235,6 +235,9 @@ mol_order int 2 y # valid options are 2 and 4 sdc_order int 2 y +# which quadrature type to use with SDC? 0 = Gauss-Lobatto, 1 = Radau +sdc_quadrature int 0 + # number of extra SDC iterations to take beyond the order sdc_extra int 0 y diff --git a/Source/driver/param_includes/castro_defaults.H b/Source/driver/param_includes/castro_defaults.H index e3b4651c45..c40cb2ad40 100644 --- a/Source/driver/param_includes/castro_defaults.H +++ b/Source/driver/param_includes/castro_defaults.H @@ -78,6 +78,7 @@ int Castro::hse_interp_temp = 0; int Castro::hse_reflect_vels = 0; int Castro::mol_order = 2; int Castro::sdc_order = 2; +int Castro::sdc_quadrature = 0; int Castro::sdc_extra = 0; int Castro::sdc_solver = 1; amrex::Real Castro::sdc_solver_tol_dens = 1.e-6; diff --git a/Source/driver/param_includes/castro_job_info_tests.H b/Source/driver/param_includes/castro_job_info_tests.H index 80b0af3c99..0d0b0d849d 100644 --- a/Source/driver/param_includes/castro_job_info_tests.H +++ b/Source/driver/param_includes/castro_job_info_tests.H @@ -73,6 +73,7 @@ jobInfoFile << (Castro::hse_interp_temp == 0 ? " " : "[*] ") << "castro.hse_i jobInfoFile << (Castro::hse_reflect_vels == 0 ? " " : "[*] ") << "castro.hse_reflect_vels = " << Castro::hse_reflect_vels << std::endl; jobInfoFile << (Castro::mol_order == 2 ? " " : "[*] ") << "castro.mol_order = " << Castro::mol_order << std::endl; jobInfoFile << (Castro::sdc_order == 2 ? " " : "[*] ") << "castro.sdc_order = " << Castro::sdc_order << std::endl; +jobInfoFile << (Castro::sdc_quadrature == 0 ? " " : "[*] ") << "castro.sdc_quadrature = " << Castro::sdc_quadrature << std::endl; jobInfoFile << (Castro::sdc_extra == 0 ? " " : "[*] ") << "castro.sdc_extra = " << Castro::sdc_extra << std::endl; jobInfoFile << (Castro::sdc_solver == 1 ? " " : "[*] ") << "castro.sdc_solver = " << Castro::sdc_solver << std::endl; jobInfoFile << (Castro::sdc_solver_tol_dens == 1.e-6 ? " " : "[*] ") << "castro.sdc_solver_tol_dens = " << Castro::sdc_solver_tol_dens << std::endl; diff --git a/Source/driver/param_includes/castro_params.H b/Source/driver/param_includes/castro_params.H index 96fc145b10..7bc371b094 100644 --- a/Source/driver/param_includes/castro_params.H +++ b/Source/driver/param_includes/castro_params.H @@ -78,6 +78,7 @@ static int hse_interp_temp; static int hse_reflect_vels; static int mol_order; static int sdc_order; +static int sdc_quadrature; static int sdc_extra; static int sdc_solver; static amrex::Real sdc_solver_tol_dens; diff --git a/Source/driver/param_includes/castro_queries.H b/Source/driver/param_includes/castro_queries.H index ead8a6a29c..12aa544d9d 100644 --- a/Source/driver/param_includes/castro_queries.H +++ b/Source/driver/param_includes/castro_queries.H @@ -78,6 +78,7 @@ pp.query("hse_interp_temp", hse_interp_temp); pp.query("hse_reflect_vels", hse_reflect_vels); pp.query("mol_order", mol_order); pp.query("sdc_order", sdc_order); +pp.query("sdc_quadrature", sdc_quadrature); pp.query("sdc_extra", sdc_extra); pp.query("sdc_solver", sdc_solver); pp.query("sdc_solver_tol_dens", sdc_solver_tol_dens); From 741b6665ea3d1a002b1adbd149d9988d0a6d213b Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 4 Sep 2019 04:42:20 -0400 Subject: [PATCH 35/72] add an sdc_quadrature parameter to optionally enable Radau --- Source/driver/Castro_setup.cpp | 68 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/Source/driver/Castro_setup.cpp b/Source/driver/Castro_setup.cpp index a0a088bb11..7e6bf63aab 100644 --- a/Source/driver/Castro_setup.cpp +++ b/Source/driver/Castro_setup.cpp @@ -1159,49 +1159,57 @@ Castro::variableSetUp () } + if (sdc_quadrature == 0) { + // Gauss-Lobatto - if (sdc_order == 2) { - // Gauss-Lobatto (trapezoid) - SDC_NODES = 2; + if (sdc_order == 2) { + // trapezoid + SDC_NODES = 2; - dt_sdc.resize(SDC_NODES); - dt_sdc = {0.0, 1.0}; + dt_sdc.resize(SDC_NODES); + dt_sdc = {0.0, 1.0}; - node_weights.resize(SDC_NODES); - node_weights = {0.5, 0.5}; + node_weights.resize(SDC_NODES); + node_weights = {0.5, 0.5}; - } else if (sdc_order == 4) { - // Gauss-Lobatto (Simpsons) - SDC_NODES = 3; + } else if (sdc_order == 4) { + // Simpsons + SDC_NODES = 3; - dt_sdc.resize(SDC_NODES); - dt_sdc = {0.0, 0.5, 1.0}; + dt_sdc.resize(SDC_NODES); + dt_sdc = {0.0, 0.5, 1.0}; - node_weights.resize(SDC_NODES); - node_weights = {1.0/6.0, 4.0/6.0, 1.0/6.0}; + node_weights.resize(SDC_NODES); + node_weights = {1.0/6.0, 4.0/6.0, 1.0/6.0}; - } else if (sdc_order == 20) { + } else { + amrex::Error("invalid value of sdc_order"); + } + + } else if (sdc_quadrature == 1) { // Radau - SDC_NODES = 3; - dt_sdc.resize(SDC_NODES); - dt_sdc = {0.0, 1.0/3.0, 1.0}; + if (sdc_order == 2) { + SDC_NODES = 3; - node_weights.resize(SDC_NODES); - node_weights = {0.0, 3.0/4.0, 1.0/4.0}; + dt_sdc.resize(SDC_NODES); + dt_sdc = {0.0, 1.0/3.0, 1.0}; - } else if (sdc_order == 40) { - // Radau - SDC_NODES = 4; + node_weights.resize(SDC_NODES); + node_weights = {0.0, 3.0/4.0, 1.0/4.0}; - dt_sdc.resize(SDC_NODES); - dt_sdc = {0.0, (4.0 - std::sqrt(6))/10.0, (4.0 + std::sqrt(6))/10.0, 1.0}; + } else if (sdc_order == 4) { + SDC_NODES = 4; - node_weights.resize(SDC_NODES); - node_weights = {0.0, (16.0 - std::sqrt(6))/36.0, (16.0 + std::sqrt(6))/36.0, 1.0/9.0}; + dt_sdc.resize(SDC_NODES); + dt_sdc = {0.0, (4.0 - std::sqrt(6))/10.0, (4.0 + std::sqrt(6))/10.0, 1.0}; - } else { - amrex::Error("invalid value of sdc_order"); - } + node_weights.resize(SDC_NODES); + node_weights = {0.0, (16.0 - std::sqrt(6))/36.0, (16.0 + std::sqrt(6))/36.0, 1.0/9.0}; + + } else { + amrex::Error("invalid value of sdc_order"); + } + } } From f681c8fec07da0214df139edabc1795d6ad150ed Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 4 Sep 2019 06:31:42 -0400 Subject: [PATCH 36/72] add sdc_quadrature to Fortran --- Source/driver/_cpp_parameters | 2 +- Source/driver/meth_params.F90 | 40 ++++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Source/driver/_cpp_parameters b/Source/driver/_cpp_parameters index d90e0b4975..3399ba8882 100644 --- a/Source/driver/_cpp_parameters +++ b/Source/driver/_cpp_parameters @@ -241,7 +241,7 @@ mol_order int 2 y sdc_order int 2 y # which quadrature type to use with SDC? 0 = Gauss-Lobatto, 1 = Radau -sdc_quadrature int 0 +sdc_quadrature int 0 y # number of extra SDC iterations to take beyond the order sdc_extra int 0 y diff --git a/Source/driver/meth_params.F90 b/Source/driver/meth_params.F90 index 6f8dbc416a..d9faf91531 100644 --- a/Source/driver/meth_params.F90 +++ b/Source/driver/meth_params.F90 @@ -164,6 +164,7 @@ module meth_params_module integer, allocatable, save :: hse_reflect_vels integer, allocatable, save :: mol_order integer, allocatable, save :: sdc_order + integer, allocatable, save :: sdc_quadrature integer, allocatable, save :: sdc_extra integer, allocatable, save :: sdc_solver real(rt), allocatable, save :: sdc_solver_tol_dens @@ -258,6 +259,7 @@ module meth_params_module attributes(managed) :: hse_reflect_vels attributes(managed) :: mol_order attributes(managed) :: sdc_order +attributes(managed) :: sdc_quadrature attributes(managed) :: sdc_extra attributes(managed) :: sdc_solver attributes(managed) :: sdc_solver_tol_dens @@ -379,6 +381,7 @@ module meth_params_module !$acc create(hse_reflect_vels) & !$acc create(mol_order) & !$acc create(sdc_order) & + !$acc create(sdc_quadrature) & !$acc create(sdc_extra) & !$acc create(sdc_solver) & !$acc create(sdc_solver_tol_dens) & @@ -625,6 +628,8 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para mol_order = 2; allocate(sdc_order) sdc_order = 2; + allocate(sdc_quadrature) + sdc_quadrature = 0; allocate(sdc_extra) sdc_extra = 0; allocate(sdc_solver) @@ -751,6 +756,7 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para call pp%query("hse_reflect_vels", hse_reflect_vels) call pp%query("mol_order", mol_order) call pp%query("sdc_order", sdc_order) + call pp%query("sdc_quadrature", sdc_quadrature) call pp%query("sdc_extra", sdc_extra) call pp%query("sdc_solver", sdc_solver) call pp%query("sdc_solver_tol_dens", sdc_solver_tol_dens) @@ -800,21 +806,22 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para !$acc device(limit_fluxes_on_small_dens, density_reset_method, allow_small_energy) & !$acc device(do_sponge, sponge_implicit, first_order_hydro) & !$acc device(hse_zero_vels, hse_interp_temp, hse_reflect_vels) & - !$acc device(mol_order, sdc_order, sdc_extra) & - !$acc device(sdc_solver, sdc_solver_tol_dens, sdc_solver_tol_spec) & - !$acc device(sdc_solver_tol_ener, sdc_solver_atol, sdc_solver_relax_factor) & - !$acc device(sdc_solve_for_rhoe, sdc_use_analytic_jac, cfl) & - !$acc device(dtnuc_e, dtnuc_X, dtnuc_X_threshold) & - !$acc device(do_react, react_T_min, react_T_max) & - !$acc device(react_rho_min, react_rho_max, disable_shock_burning) & - !$acc device(T_guess, diffuse_temp, diffuse_cutoff_density) & - !$acc device(diffuse_cutoff_density_hi, diffuse_cond_scale_fac, do_grav) & - !$acc device(grav_source_type, do_rotation, rot_period) & - !$acc device(rot_period_dot, rotation_include_centrifugal, rotation_include_coriolis) & - !$acc device(rotation_include_domegadt, state_in_rotating_frame, rot_source_type) & - !$acc device(implicit_rotation_update, rot_axis, use_point_mass) & - !$acc device(point_mass, point_mass_fix_solution, do_acc) & - !$acc device(grown_factor, track_grid_losses, const_grav, get_g_from_phi) + !$acc device(mol_order, sdc_order, sdc_quadrature) & + !$acc device(sdc_extra, sdc_solver, sdc_solver_tol_dens) & + !$acc device(sdc_solver_tol_spec, sdc_solver_tol_ener, sdc_solver_atol) & + !$acc device(sdc_solver_relax_factor, sdc_solve_for_rhoe, sdc_use_analytic_jac) & + !$acc device(cfl, dtnuc_e, dtnuc_X) & + !$acc device(dtnuc_X_threshold, do_react, react_T_min) & + !$acc device(react_T_max, react_rho_min, react_rho_max) & + !$acc device(disable_shock_burning, T_guess, diffuse_temp) & + !$acc device(diffuse_cutoff_density, diffuse_cutoff_density_hi, diffuse_cond_scale_fac) & + !$acc device(do_grav, grav_source_type, do_rotation) & + !$acc device(rot_period, rot_period_dot, rotation_include_centrifugal) & + !$acc device(rotation_include_coriolis, rotation_include_domegadt, state_in_rotating_frame) & + !$acc device(rot_source_type, implicit_rotation_update, rot_axis) & + !$acc device(use_point_mass, point_mass, point_mass_fix_solution) & + !$acc device(do_acc, grown_factor, track_grid_losses) & + !$acc device(const_grav, get_g_from_phi) #ifdef GRAVITY @@ -1055,6 +1062,9 @@ subroutine ca_finalize_meth_params() bind(C, name="ca_finalize_meth_params") if (allocated(sdc_order)) then deallocate(sdc_order) end if + if (allocated(sdc_quadrature)) then + deallocate(sdc_quadrature) + end if if (allocated(sdc_extra)) then deallocate(sdc_extra) end if From a0622540428762843ed515f1d4e40083407976a8 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 5 Sep 2019 06:25:21 -0400 Subject: [PATCH 37/72] define dt_m more generally --- Source/driver/Castro_sdc.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Source/driver/Castro_sdc.cpp b/Source/driver/Castro_sdc.cpp index dd4b8c8174..7426f51cff 100644 --- a/Source/driver/Castro_sdc.cpp +++ b/Source/driver/Castro_sdc.cpp @@ -33,6 +33,9 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { MultiFab tmp; MultiFab& C_source = (sdc_order == 4) ? get_new_data(SDC_Source_Type) : tmp; + // the timestep from m to m+1 + Real dt_m = (dt_sdc[m_end] - dt_sdc[m_start]) * dt; + if (sdc_order == 4) { // for 4th order reacting flow, we need to create the "source" C @@ -66,8 +69,6 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { // staging place so we can do a FillPatch MultiFab& S_new = get_new_data(State_Type); - Real dt_m = dt/2.0; - for (MFIter mfi(S_new); mfi.isValid(); ++mfi) { const Box& bx = mfi.tilebox(); @@ -121,9 +122,6 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { &m_start); } else { - // the timestep from m to m+1 - Real dt_m = dt/2.0; - // fourth order SDC reaction update -- we need to respect the // difference between cell-centers and averages From b0db99cc1aa89ce50c0c573c2a9914f973cab273 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 5 Sep 2019 07:01:32 -0400 Subject: [PATCH 38/72] some more straightening out of dt vs dt_m --- Source/driver/Castro_sdc.cpp | 12 +-- Source/driver/sdc_util.F90 | 145 ++++++++++++++++++++++------------- 2 files changed, 98 insertions(+), 59 deletions(-) diff --git a/Source/driver/Castro_sdc.cpp b/Source/driver/Castro_sdc.cpp index 7426f51cff..caea8337df 100644 --- a/Source/driver/Castro_sdc.cpp +++ b/Source/driver/Castro_sdc.cpp @@ -28,14 +28,14 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { const int* domain_lo = geom.Domain().loVect(); const int* domain_hi = geom.Domain().hiVect(); + // the timestep from m to m+1 + Real dt_m = (dt_sdc[m_end] - dt_sdc[m_start]) * dt; + #ifdef REACTIONS // SDC_Source_Type is only defined for 4th order MultiFab tmp; MultiFab& C_source = (sdc_order == 4) ? get_new_data(SDC_Source_Type) : tmp; - // the timestep from m to m+1 - Real dt_m = (dt_sdc[m_end] - dt_sdc[m_start]) * dt; - if (sdc_order == 4) { // for 4th order reacting flow, we need to create the "source" C @@ -110,7 +110,7 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { // second order SDC reaction update -- we don't care about // the difference between cell-centers and averages - ca_sdc_update_o2(BL_TO_FORTRAN_BOX(bx), &dt, + ca_sdc_update_o2(BL_TO_FORTRAN_BOX(bx), &dt_m, BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), @@ -180,7 +180,7 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { #else // pure advection if (sdc_order == 2) { - ca_sdc_update_advection_o2(BL_TO_FORTRAN_BOX(bx), &dt, + ca_sdc_update_advection_o2(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), @@ -188,7 +188,7 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { BL_TO_FORTRAN_3D((*A_old[1])[mfi]), &m_start); } else { - ca_sdc_update_advection_o4(BL_TO_FORTRAN_BOX(bx), &dt, + ca_sdc_update_advection_o4(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 2dec54fbe4..fc6c3baa81 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -828,7 +828,7 @@ subroutine f_sdc_jac(n, U, f, Jac, ldjac, iflag, rpar) end subroutine f_sdc_jac #endif - subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, & + subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, dt, & k_m, kmlo, kmhi, & k_n, knlo, knhi, & A_m, Amlo, Amhi, & @@ -837,13 +837,16 @@ subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, & m_start) bind(C, name="ca_sdc_update_advection_o2") ! update k_m to k_n via advection -- this is a second-order accurate update + ! here, dt_m is the update for this stage, from one time node to the next + ! dt is the update over the whole timestep, n to n+1 + use meth_params_module, only : NVAR use amrex_constants_module, only : HALF implicit none integer, intent(in) :: lo(3), hi(3) - real(rt), intent(in) :: dt_m + real(rt), intent(in) :: dt_m, dt integer, intent(in) :: kmlo(3), kmhi(3) integer, intent(in) :: knlo(3), knhi(3) integer, intent(in) :: Amlo(3), Amhi(3) @@ -861,18 +864,30 @@ subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, & integer :: i, j, k - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + HALF * dt_m * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) - enddo - enddo - enddo + if (sdc_quadrature == 0) then + + ! Gauss-Lobatto / trapezoid + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + HALF * dt * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + end do + end do + end do + + else if (sdc_quadrature == 1) then + + ! Radau + call amrex_error("2nd order Radau not implemented") + + end if + end subroutine ca_sdc_update_advection_o2 - subroutine ca_sdc_update_advection_o4(lo, hi, dt, & + subroutine ca_sdc_update_advection_o4(lo, hi, dt_m, dt, & k_m, kmlo, kmhi, & k_n, knlo, knhi, & A_m, Amlo, Amhi, & @@ -881,7 +896,9 @@ subroutine ca_sdc_update_advection_o4(lo, hi, dt, & A_2_old, A2lo, A2hi, & m_start) bind(C, name="ca_sdc_update_advection_o4") ! update k_m to k_n via advection -- this is a second-order accurate update - ! dt is the total timestep from n to n+1 + + ! here, dt_m is the update for this stage, from one time node to the next + ! dt is the update over the whole timestep, n to n+1 use meth_params_module, only : NVAR use amrex_constants_module, only : HALF, TWO, FIVE, EIGHT @@ -889,7 +906,7 @@ subroutine ca_sdc_update_advection_o4(lo, hi, dt, & implicit none integer, intent(in) :: lo(3), hi(3) - real(rt), intent(in) :: dt + real(rt), intent(in) :: dt_m, dt integer, intent(in) :: kmlo(3), kmhi(3) integer, intent(in) :: knlo(3), knhi(3) integer, intent(in) :: Amlo(3), Amhi(3) @@ -908,38 +925,45 @@ subroutine ca_sdc_update_advection_o4(lo, hi, dt, & real(rt), intent(in) :: A_2_old(A2lo(1):A2hi(1), A2lo(2):A2hi(2), A2lo(3):A2hi(3), NVAR) integer :: i, j, k - real(rt) :: dt_m + if (sdc_quadrature == 0) then - dt_m = HALF * dt + ! Gauss-Lobatto (Simpsons) - if (m_start == 0) then + if (m_start == 0) then - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + & - dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & - dt/24.0_rt * (FIVE*A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) - A_2_old(i,j,k,:)) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & + dt/24.0_rt * (FIVE*A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) - A_2_old(i,j,k,:)) + enddo enddo enddo - enddo - else if (m_start == 1) then + else if (m_start == 1) then - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + & - dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & - dt/24.0_rt * (-A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) + FIVE*A_2_old(i,j,k,:)) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & + dt/24.0_rt * (-A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) + FIVE*A_2_old(i,j,k,:)) + enddo enddo enddo - enddo - else - call castro_error("error in ca_sdc_update_advection_o4 -- should not be here") - endif + else + call castro_error("error in ca_sdc_update_advection_o4 -- should not be here") + endif + + else if (sdc_quadrature == 1) then + + ! Radau + call amrex_error("4th order Radau not implemented") + + end if end subroutine ca_sdc_update_advection_o4 @@ -986,33 +1010,44 @@ subroutine ca_sdc_compute_C4(lo, hi, & integer :: i, j, k real(rt) :: integral(NVAR) - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) + if (sdc_quadrature == 0) then - ! compute the integral (without the dt). Note that each of these is over - ! dt/2 - if (m_start == 0) then - integral(:) = ONE/12.0_rt * (FIVE*(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & - EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) - & - (A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) + ! Gauss-Lobatto (Simpsons) - C(i,j,k,:) = (A_m(i,j,k,:) - A_0_old(i,j,k,:)) - R_1_old(i,j,k,:) + integral + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) - else if (m_start == 1) then - integral(:) = ONE/12.0_rt * (-(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & - EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & - FIVE*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) + ! compute the integral (without the dt). Note that each of these is over + ! dt/2 + if (m_start == 0) then + integral(:) = ONE/12.0_rt * (FIVE*(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & + EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) - & + (A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) - C(i,j,k,:) = (A_m(i,j,k,:) - A_1_old(i,j,k,:)) - R_2_old(i,j,k,:) + integral + C(i,j,k,:) = (A_m(i,j,k,:) - A_0_old(i,j,k,:)) - R_1_old(i,j,k,:) + integral - else - call castro_error("error in ca_sdc_compute_C4 -- should not be here") - endif + else if (m_start == 1) then + integral(:) = ONE/12.0_rt * (-(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & + EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + FIVE*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) + + C(i,j,k,:) = (A_m(i,j,k,:) - A_1_old(i,j,k,:)) - R_2_old(i,j,k,:) + integral + else + call castro_error("error in ca_sdc_compute_C4 -- should not be here") + endif + + enddo enddo enddo - enddo + + else + + ! Radau + call amrex_error("Error: Radau not implemented for ca_sdc_compute_C4") + + end if end subroutine ca_sdc_compute_C4 @@ -1046,6 +1081,8 @@ subroutine ca_sdc_compute_initial_guess(lo, hi, & integer, intent(in) :: sdc_iteration integer :: i, j, k + ! Here dt_m is the timestep to update from time node m to m+1 + do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -1075,6 +1112,8 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & m_start) bind(C, name="ca_sdc_update_o2") ! update k_m to k_n via advection -- this is a second-order accurate update + ! Here, dt_m is the timestep between time-nodes m and m+1 + use meth_params_module, only : NVAR use amrex_constants_module, only : ZERO, HALF use burn_type_module, only : burn_t @@ -1174,7 +1213,7 @@ subroutine ca_sdc_update_centers_o4(lo, hi, dt_m, & ! update U_old to U_new on cell-centers. This is an implicit ! solve because of reactions. Here U_old corresponds to time node ! m and U_new is node m+1. dt_m is the timestep between m and - ! m+1, which is dt_m = dt/2. + ! m+1 use meth_params_module, only : NVAR use react_util_module, only : okay_to_burn From 95fa608347d9d17630e4c8fae817b61abb00fc82 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 5 Sep 2019 09:07:43 -0400 Subject: [PATCH 39/72] some work for pure hydro Radau SDC --- Source/driver/Castro_sdc.cpp | 58 ++++++-- Source/driver/sdc_util.F90 | 252 +++++++++++++++++++++++++++------- Source/hydro/Castro_hydro_F.H | 35 ++++- 3 files changed, 276 insertions(+), 69 deletions(-) diff --git a/Source/driver/Castro_sdc.cpp b/Source/driver/Castro_sdc.cpp index caea8337df..603228fbc5 100644 --- a/Source/driver/Castro_sdc.cpp +++ b/Source/driver/Castro_sdc.cpp @@ -180,22 +180,50 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { #else // pure advection if (sdc_order == 2) { - ca_sdc_update_advection_o2(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, - BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), - BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*A_old[0])[mfi]), - BL_TO_FORTRAN_3D((*A_old[1])[mfi]), - &m_start); + + if (sdc_quadrature == 0) { + ca_sdc_update_advection_o2_lobatto(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, + BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + &m_start); + + } else { + ca_sdc_update_advection_o2_radau(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, + BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + &m_start); + + } + } else { - ca_sdc_update_advection_o4(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, - BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), - BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*A_old[0])[mfi]), - BL_TO_FORTRAN_3D((*A_old[1])[mfi]), - BL_TO_FORTRAN_3D((*A_old[2])[mfi]), - &m_start); + if (sdc_quadrature == 0) { + ca_sdc_update_advection_o4_lobatto(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, + BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + &m_start); + } else { + ca_sdc_update_advection_o4_radau(BL_TO_FORTRAN_BOX(bx), &dt_m, &dt, + BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + BL_TO_FORTRAN_3D((*A_old[3])[mfi]), + &m_start); + } + } #endif diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index fc6c3baa81..274e3c92ed 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -828,14 +828,15 @@ subroutine f_sdc_jac(n, U, f, Jac, ldjac, iflag, rpar) end subroutine f_sdc_jac #endif - subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, dt, & - k_m, kmlo, kmhi, & - k_n, knlo, knhi, & - A_m, Amlo, Amhi, & - A_0_old, A0lo, A0hi, & - A_1_old, A1lo, A1hi, & - m_start) bind(C, name="ca_sdc_update_advection_o2") + subroutine ca_sdc_update_advection_o2_lobatto(lo, hi, dt_m, dt, & + k_m, kmlo, kmhi, & + k_n, knlo, knhi, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + m_start) bind(C, name="ca_sdc_update_advection_o2_lobatto") ! update k_m to k_n via advection -- this is a second-order accurate update + ! for the Gauss-Lobatto discretization of the time nodes ! here, dt_m is the update for this stage, from one time node to the next ! dt is the update over the whole timestep, n to n+1 @@ -864,38 +865,103 @@ subroutine ca_sdc_update_advection_o2(lo, hi, dt_m, dt, & integer :: i, j, k - if (sdc_quadrature == 0) then + ! Gauss-Lobatto / trapezoid + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + HALF * dt * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + end do + end do + end do + + end subroutine ca_sdc_update_advection_o2_lobatto + + + subroutine ca_sdc_update_advection_o2_radau(lo, hi, dt_m, dt, & + k_m, kmlo, kmhi, & + k_n, knlo, knhi, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + m_start) bind(C, name="ca_sdc_update_advection_o2_radau") + ! update k_m to k_n via advection -- this is a second-order accurate update + ! for the Radau discretization of the time nodes + + ! here, dt_m is the update for this stage, from one time node to the next + ! dt is the update over the whole timestep, n to n+1 + + use meth_params_module, only : NVAR + use amrex_constants_module, only : HALF, FIVE + + implicit none + + integer, intent(in) :: lo(3), hi(3) + real(rt), intent(in) :: dt_m, dt + integer, intent(in) :: kmlo(3), kmhi(3) + integer, intent(in) :: knlo(3), knhi(3) + integer, intent(in) :: Amlo(3), Amhi(3) + integer, intent(in) :: A0lo(3), A0hi(3) + integer, intent(in) :: A1lo(3), A1hi(3) + integer, intent(in) :: A2lo(3), A2hi(3) + integer, intent(in) :: m_start + + + real(rt), intent(in) :: k_m(kmlo(1):kmhi(1), kmlo(2):kmhi(2), kmlo(3):kmhi(3), NVAR) + real(rt), intent(inout) :: k_n(knlo(1):knhi(1), knlo(2):knhi(2), knlo(3):knhi(3), NVAR) + + real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) + real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) + real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) + real(rt), intent(in) :: A_2_old(A2lo(1):A2hi(1), A2lo(2):A2hi(2), A2lo(3):A2hi(3), NVAR) + + integer :: i, j, k - ! Gauss-Lobatto / trapezoid + ! Radau + + if (m_start == 0) then do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + HALF * dt * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & + dt/12.0_rt * (FIVE*A_1_old(i,j,k,:) - A_2_old(i,j,k,:)) + end do end do end do - else if (sdc_quadrature == 1) then + else if (m_start == 1) then - ! Radau - call amrex_error("2nd order Radau not implemented") + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) - end if + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & + dt/3.0_rt * (A_1_old(i,j,k,:) + A_2_old(i,j,k,:)) + end do + end do + end do + + end if - end subroutine ca_sdc_update_advection_o2 + end subroutine ca_sdc_update_advection_o2_radau - subroutine ca_sdc_update_advection_o4(lo, hi, dt_m, dt, & - k_m, kmlo, kmhi, & - k_n, knlo, knhi, & - A_m, Amlo, Amhi, & - A_0_old, A0lo, A0hi, & - A_1_old, A1lo, A1hi, & - A_2_old, A2lo, A2hi, & - m_start) bind(C, name="ca_sdc_update_advection_o4") - ! update k_m to k_n via advection -- this is a second-order accurate update + subroutine ca_sdc_update_advection_o4_lobatto(lo, hi, dt_m, dt, & + k_m, kmlo, kmhi, & + k_n, knlo, knhi, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + m_start) bind(C, name="ca_sdc_update_advection_o4_lobatto") + ! update k_m to k_n via advection -- this is a fourth order accurate update ! here, dt_m is the update for this stage, from one time node to the next ! dt is the update over the whole timestep, n to n+1 @@ -926,46 +992,132 @@ subroutine ca_sdc_update_advection_o4(lo, hi, dt_m, dt, & integer :: i, j, k - if (sdc_quadrature == 0) then + ! Gauss-Lobatto (Simpsons) - ! Gauss-Lobatto (Simpsons) + if (m_start == 0) then - if (m_start == 0) then + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & + dt/24.0_rt * (FIVE*A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) - A_2_old(i,j,k,:)) + enddo + enddo + enddo - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + & - dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & - dt/24.0_rt * (FIVE*A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) - A_2_old(i,j,k,:)) - enddo + else if (m_start == 1) then + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & + dt/24.0_rt * (-A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) + FIVE*A_2_old(i,j,k,:)) enddo enddo + enddo + + else + call castro_error("error in ca_sdc_update_advection_o4_lobatto -- should not be here") + endif + + end subroutine ca_sdc_update_advection_o4_lobatto + + + subroutine ca_sdc_update_advection_o4_radau(lo, hi, dt_m, dt, & + k_m, kmlo, kmhi, & + k_n, knlo, knhi, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + A_3_old, A3lo, A3hi, & + m_start) bind(C, name="ca_sdc_update_advection_o4_radau") + ! update k_m to k_n via advection -- this is a fourth-order accurate update + + ! here, dt_m is the update for this stage, from one time node to the next + ! dt is the update over the whole timestep, n to n+1 + + use meth_params_module, only : NVAR + use amrex_constants_module, only : HALF, TWO, FIVE, EIGHT + + implicit none + + integer, intent(in) :: lo(3), hi(3) + real(rt), intent(in) :: dt_m, dt + integer, intent(in) :: kmlo(3), kmhi(3) + integer, intent(in) :: knlo(3), knhi(3) + integer, intent(in) :: Amlo(3), Amhi(3) + integer, intent(in) :: A0lo(3), A0hi(3) + integer, intent(in) :: A1lo(3), A1hi(3) + integer, intent(in) :: A2lo(3), A2hi(3) + integer, intent(in) :: A3lo(3), A3hi(3) + integer, intent(in) :: m_start + - else if (m_start == 1) then + real(rt), intent(in) :: k_m(kmlo(1):kmhi(1), kmlo(2):kmhi(2), kmlo(3):kmhi(3), NVAR) + real(rt), intent(inout) :: k_n(knlo(1):knhi(1), knlo(2):knhi(2), knlo(3):knhi(3), NVAR) - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - k_n(i,j,k,:) = k_m(i,j,k,:) + & - dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & - dt/24.0_rt * (-A_0_old(i,j,k,:) + EIGHT*A_1_old(i,j,k,:) + FIVE*A_2_old(i,j,k,:)) - enddo + real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) + real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) + real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) + real(rt), intent(in) :: A_2_old(A2lo(1):A2hi(1), A2lo(2):A2hi(2), A2lo(3):A2hi(3), NVAR) + real(rt), intent(in) :: A_3_old(A3lo(1):A3hi(1), A3lo(2):A3hi(2), A3lo(3):A3hi(3), NVAR) + + integer :: i, j, k + + ! Gauss-Lobatto (Simpsons) + + if (m_start == 0) then + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & + dt/1800.0_rt * ((-35.0_rt*sqrt(6.0_rt) + 440.0_rt)*A_1_old(i,j,k,:) + & + (-169.0_rt*sqrt(6.0_rt) + 296.0_rt)*A_2_old(i,j,k,:) + & + (-16.0_rt + 24.0_rt*sqrt(6.0_rt))*A_3_old(i,j,k,:)) enddo enddo + enddo - else - call castro_error("error in ca_sdc_update_advection_o4 -- should not be here") - endif + else if (m_start == 1) then - else if (sdc_quadrature == 1) then + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & + dt/150.0_rt * ((-12.0_rt + 17.0_rt*sqrt(6.0_rt))*A_1_old(i,j,k,:) + & + (12.0_rt + 17.0_rt*sqrt(6.0_rt))*A_2_old(i,j,k,:) + & + (-4.0_rt*sqrt(6.0_rt))*A_3_old(i,j,k,:)) + enddo + enddo + enddo - ! Radau - call amrex_error("4th order Radau not implemented") + else if (m_start == 2) then + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + k_n(i,j,k,:) = k_m(i,j,k,:) + & + dt_m * (A_m(i,j,k,:) - A_2_old(i,j,k,:)) + & + dt/600.0_rt * ((168.0_rt - 73.0_rt*sqrt(6.0_rt))*A_1_old(i,j,k,:) + & + (120.0_rt + 5.0_rt*sqrt(6.0_rt))*A_2_old(i,j,k,:) + & + (72.0_rt + 8.0_rt*sqrt(6.0_rt))*A_3_old(i,j,k,:)) + enddo + enddo + enddo + + else + call castro_error("error in ca_sdc_update_advection_o4_radau -- should not be here") + endif + + end subroutine ca_sdc_update_advection_o4_radau - end if - end subroutine ca_sdc_update_advection_o4 #ifdef REACTIONS diff --git a/Source/hydro/Castro_hydro_F.H b/Source/hydro/Castro_hydro_F.H index 17546fc893..65f19998c2 100644 --- a/Source/hydro/Castro_hydro_F.H +++ b/Source/hydro/Castro_hydro_F.H @@ -503,9 +503,10 @@ extern "C" const amrex::Real* dx); #ifndef AMREX_USE_CUDA - void ca_sdc_update_advection_o2 + void ca_sdc_update_advection_o2_lobatto (const int* lo, const int* hi, - const amrex::Real* time, + const amrex::Real* dt_m, + const amrex::Real* dt, const BL_FORT_FAB_ARG_3D(k_m), BL_FORT_FAB_ARG_3D(k_n), BL_FORT_FAB_ARG_3D(A_m), @@ -513,15 +514,41 @@ extern "C" BL_FORT_FAB_ARG_3D(A_1_old), const int* m_start); - void ca_sdc_update_advection_o4 + void ca_sdc_update_advection_o2_radau (const int* lo, const int* hi, - const amrex::Real* time, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(k_m), + BL_FORT_FAB_ARG_3D(k_n), + BL_FORT_FAB_ARG_3D(A_m), + BL_FORT_FAB_ARG_3D(A_0_old), + BL_FORT_FAB_ARG_3D(A_1_old), + BL_FORT_FAB_ARG_3D(A_2_old), + const int* m_start); + + void ca_sdc_update_advection_o4_lobatto + (const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(k_m), + BL_FORT_FAB_ARG_3D(k_n), + BL_FORT_FAB_ARG_3D(A_m), + BL_FORT_FAB_ARG_3D(A_0_old), + BL_FORT_FAB_ARG_3D(A_1_old), + BL_FORT_FAB_ARG_3D(A_2_old), + const int* m_start); + + void ca_sdc_update_advection_o4_radau + (const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, const BL_FORT_FAB_ARG_3D(k_m), BL_FORT_FAB_ARG_3D(k_n), BL_FORT_FAB_ARG_3D(A_m), BL_FORT_FAB_ARG_3D(A_0_old), BL_FORT_FAB_ARG_3D(A_1_old), BL_FORT_FAB_ARG_3D(A_2_old), + BL_FORT_FAB_ARG_3D(A_3_old), const int* m_start); #ifdef REACTIONS From c4ce5522d04d1445c5f54bf29eddbfb5ed60f23e Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 5 Sep 2019 10:05:07 -0400 Subject: [PATCH 40/72] compute the Radau integrals for reaction --- Source/driver/sdc_util.F90 | 162 ++++++++++++++++++++++++++++--------- 1 file changed, 123 insertions(+), 39 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 274e3c92ed..edb05331e8 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1118,19 +1118,18 @@ subroutine ca_sdc_update_advection_o4_radau(lo, hi, dt_m, dt, & end subroutine ca_sdc_update_advection_o4_radau - - #ifdef REACTIONS - subroutine ca_sdc_compute_C4(lo, hi, & - A_m, Amlo, Amhi, & - A_0_old, A0lo, A0hi, & - A_1_old, A1lo, A1hi, & - A_2_old, A2lo, A2hi, & - R_0_old, R0lo, R0hi, & - R_1_old, R1lo, R1hi, & - R_2_old, R2lo, R2hi, & - C, Clo, Chi, & - m_start) bind(C, name="ca_sdc_compute_C4") + subroutine ca_sdc_compute_C4_lobatto(lo, hi, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + R_0_old, R0lo, R0hi, & + R_1_old, R1lo, R1hi, & + R_2_old, R2lo, R2hi, & + C, Clo, Chi, & + m_start) bind(C, name="ca_sdc_compute_C4_lobatto") + ! compute the 'C' term for the 4th-order solve with reactions ! note: this 'C' is cell-averages @@ -1162,46 +1161,131 @@ subroutine ca_sdc_compute_C4(lo, hi, & integer :: i, j, k real(rt) :: integral(NVAR) - if (sdc_quadrature == 0) then - - ! Gauss-Lobatto (Simpsons) + ! Gauss-Lobatto (Simpsons) - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) - ! compute the integral (without the dt). Note that each of these is over - ! dt/2 - if (m_start == 0) then - integral(:) = ONE/12.0_rt * (FIVE*(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & - EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) - & - (A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) + if (m_start == 0) then + ! compute the integral from [t_m, t_{m+1}], normalized by dt_m + integral(:) = ONE/12.0_rt * (FIVE*(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & + EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) - & + (A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) - C(i,j,k,:) = (A_m(i,j,k,:) - A_0_old(i,j,k,:)) - R_1_old(i,j,k,:) + integral + C(i,j,k,:) = (A_m(i,j,k,:) - A_0_old(i,j,k,:)) - R_1_old(i,j,k,:) + integral - else if (m_start == 1) then - integral(:) = ONE/12.0_rt * (-(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & - EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & - FIVE*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) + else if (m_start == 1) then + ! compute the integral from [t_m, t_{m+1}], normalized by dt_m + integral(:) = ONE/12.0_rt * (-(A_0_old(i,j,k,:) + R_0_old(i,j,k,:)) + & + EIGHT*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + FIVE*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:))) - C(i,j,k,:) = (A_m(i,j,k,:) - A_1_old(i,j,k,:)) - R_2_old(i,j,k,:) + integral + C(i,j,k,:) = (A_m(i,j,k,:) - A_1_old(i,j,k,:)) - R_2_old(i,j,k,:) + integral - else - call castro_error("error in ca_sdc_compute_C4 -- should not be here") - endif + else + call castro_error("error in ca_sdc_compute_C4 -- should not be here") + endif - enddo enddo enddo + enddo - else + end subroutine ca_sdc_compute_C4_lobatto + + + subroutine ca_sdc_compute_C4_radau(lo, hi, dt_m, dt, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + A_3_old, A3lo, A3hi, & + R_0_old, R0lo, R0hi, & + R_1_old, R1lo, R1hi, & + R_2_old, R2lo, R2hi, & + R_3_old, R3lo, R3hi, & + C, Clo, Chi, & + m_start) bind(C, name="ca_sdc_compute_C4_radau") + ! compute the 'C' term for the 4th-order solve with reactions + ! note: this 'C' is cell-averages - ! Radau - call amrex_error("Error: Radau not implemented for ca_sdc_compute_C4") + use meth_params_module, only : NVAR + use amrex_constants_module, only : ONE, HALF, TWO, FIVE, EIGHT - end if + implicit none + + integer, intent(in) :: lo(3), hi(3) + integer, intent(in) :: Amlo(3), Amhi(3) + integer, intent(in) :: A0lo(3), A0hi(3) + integer, intent(in) :: A1lo(3), A1hi(3) + integer, intent(in) :: A2lo(3), A2hi(3) + integer, intent(in) :: A3lo(3), A3hi(3) + integer, intent(in) :: R0lo(3), R0hi(3) + integer, intent(in) :: R1lo(3), R1hi(3) + integer, intent(in) :: R2lo(3), R2hi(3) + integer, intent(in) :: R3lo(3), R3hi(3) + integer, intent(in) :: Clo(3), Chi(3) + integer, intent(in) :: m_start + + real(rt), intent(in) :: dt_m, dt + real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) + real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) + real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) + real(rt), intent(in) :: A_2_old(A2lo(1):A2hi(1), A2lo(2):A2hi(2), A2lo(3):A2hi(3), NVAR) + real(rt), intent(in) :: A_3_old(A3lo(1):A3hi(1), A3lo(2):A3hi(2), A3lo(3):A3hi(3), NVAR) + real(rt), intent(in) :: R_0_old(R0lo(1):R0hi(1), R0lo(2):R0hi(2), R0lo(3):R0hi(3), NVAR) + real(rt), intent(in) :: R_1_old(R1lo(1):R1hi(1), R1lo(2):R1hi(2), R1lo(3):R1hi(3), NVAR) + real(rt), intent(in) :: R_2_old(R2lo(1):R2hi(1), R2lo(2):R2hi(2), R2lo(3):R2hi(3), NVAR) + real(rt), intent(in) :: R_3_old(R3lo(1):R3hi(1), R3lo(2):R3hi(2), R3lo(3):R3hi(3), NVAR) + real(rt), intent(out) :: C(Clo(1):Chi(1), Clo(2):Chi(2), Clo(3):Chi(3), NVAR) + + integer :: i, j, k + real(rt) :: integral(NVAR) + + ! Gauss-Lobatto (Simpsons) + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + if (m_start == 0) then + ! compute the integral from [t_m, t_{m+1}], normalized by dt_m + integral(:) = (dt/dt_m) * (ONE/1800.0_rt) * & + ((-35.0_rt*sqrt(6.0_rt) + 440.0_rt)*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + (-169.0_rt*sqrt(6.0_rt) + 296.0_rt)*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:)) + & + (-16.0_rt + 24.0_rt*sqrt(6.0_rt))*(A_3_old(i,j,k,:) + R_3_old(i,j,k,:))) + + C(i,j,k,:) = (A_m(i,j,k,:) - A_0_old(i,j,k,:)) - R_1_old(i,j,k,:) + integral + + else if (m_start == 1) then + ! compute the integral from [t_m, t_{m+1}], normalized by dt_m + integral(:) = (dt/dt_m) * (ONE/150.0_rt) * & + ((-12.0_rt + 17.0_rt*sqrt(6.0_rt))*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + (12.0_rt + 17.0_rt*sqrt(6.0_rt))*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:)) + & + (-4.0_rt*sqrt(6.0_rt))*(A_3_old(i,j,k,:) + R_3_old(i,j,k,:))) + + C(i,j,k,:) = (A_m(i,j,k,:) - A_1_old(i,j,k,:)) - R_2_old(i,j,k,:) + integral + + + else if (m_start == 2) then + ! compute the integral from [t_m, t_{m+1}], normalized by dt_m + integral(:) = (dt/dt_m) * (ONE/600.0_rt) * & + ((168.0_rt - 73.0_rt*sqrt(6.0_rt))*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + (120.0_rt + 5.0_rt*sqrt(6.0_rt))*(A_2_old(i,j,k,:) + R_2_old(i,j,k,:)) + & + (72.0_rt + 8.0_rt*sqrt(6.0_rt))*(A_3_old(i,j,k,:) + R_3_old(i,j,k,:))) + + C(i,j,k,:) = (A_m(i,j,k,:) - A_2_old(i,j,k,:)) - R_3_old(i,j,k,:) + integral + + else + call castro_error("error in ca_sdc_compute_C4 -- should not be here") + endif + + end do + end do + end do + + end subroutine ca_sdc_compute_C4_radau - end subroutine ca_sdc_compute_C4 subroutine ca_sdc_compute_initial_guess(lo, hi, & U_old, Uo_lo, Uo_hi, & From 683afe5ba0edb7608e244b453379ee5e7032f311 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 5 Sep 2019 17:57:26 -0400 Subject: [PATCH 41/72] finish first pass at implementing Radau --- Source/driver/Castro_sdc.cpp | 78 ++++++++++++--- Source/driver/sdc_util.F90 | 177 +++++++++++++++++++++++++++++----- Source/hydro/Castro_hydro_F.H | 73 +++++++++++--- 3 files changed, 272 insertions(+), 56 deletions(-) diff --git a/Source/driver/Castro_sdc.cpp b/Source/driver/Castro_sdc.cpp index 603228fbc5..c3a7e3d843 100644 --- a/Source/driver/Castro_sdc.cpp +++ b/Source/driver/Castro_sdc.cpp @@ -45,16 +45,34 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { const Box& bx = mfi.tilebox(); - ca_sdc_compute_C4(BL_TO_FORTRAN_BOX(bx), - BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*A_old[0])[mfi]), - BL_TO_FORTRAN_3D((*A_old[1])[mfi]), - BL_TO_FORTRAN_3D((*A_old[2])[mfi]), - BL_TO_FORTRAN_3D((*R_old[0])[mfi]), - BL_TO_FORTRAN_3D((*R_old[1])[mfi]), - BL_TO_FORTRAN_3D((*R_old[2])[mfi]), - BL_TO_FORTRAN_3D(C_source[mfi]), - &m_start); + if (sdc_quadrature == 0) { + + ca_sdc_compute_C4_lobatto(BL_TO_FORTRAN_BOX(bx), + &dt_m, &dt, + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + BL_TO_FORTRAN_3D((*R_old[0])[mfi]), + BL_TO_FORTRAN_3D((*R_old[1])[mfi]), + BL_TO_FORTRAN_3D((*R_old[2])[mfi]), + BL_TO_FORTRAN_3D(C_source[mfi]), + &m_start); + } else { + ca_sdc_compute_C4_radau(BL_TO_FORTRAN_BOX(bx), + &dt_m, &dt, + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + BL_TO_FORTRAN_3D((*A_old[3])[mfi]), + BL_TO_FORTRAN_3D((*R_old[0])[mfi]), + BL_TO_FORTRAN_3D((*R_old[1])[mfi]), + BL_TO_FORTRAN_3D((*R_old[2])[mfi]), + BL_TO_FORTRAN_3D((*R_old[3])[mfi]), + BL_TO_FORTRAN_3D(C_source[mfi]), + &m_start); + } } // need to construct the time for this stage -- but it is not really @@ -98,6 +116,8 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { FArrayBox U_new_center; FArrayBox R_new; + FArrayBox C2; + for (MFIter mfi(*k_new[0]); mfi.isValid(); ++mfi) { const Box& bx = mfi.tilebox(); @@ -110,14 +130,44 @@ Castro::do_sdc_update(int m_start, int m_end, Real dt) { // second order SDC reaction update -- we don't care about // the difference between cell-centers and averages + // first compute the source term, C -- this differs depending + // on whether we are Lobatto or Radau + C2.resize(bx, NUM_STATE); + + if (sdc_quadrature == 0) { + + ca_sdc_compute_C2_lobatto(BL_TO_FORTRAN_BOX(bx), + &dt_m, &dt, + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*R_old[0])[mfi]), + BL_TO_FORTRAN_3D((*R_old[1])[mfi]), + BL_TO_FORTRAN_3D(C2), + &m_start); + + } else { + + ca_sdc_compute_C2_radau(BL_TO_FORTRAN_BOX(bx), + &dt_m, &dt, + BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), + BL_TO_FORTRAN_3D((*A_old[0])[mfi]), + BL_TO_FORTRAN_3D((*A_old[1])[mfi]), + BL_TO_FORTRAN_3D((*A_old[2])[mfi]), + BL_TO_FORTRAN_3D((*R_old[0])[mfi]), + BL_TO_FORTRAN_3D((*R_old[1])[mfi]), + BL_TO_FORTRAN_3D((*R_old[2])[mfi]), + BL_TO_FORTRAN_3D(C2), + &m_start); + + } + ca_sdc_update_o2(BL_TO_FORTRAN_BOX(bx), &dt_m, BL_TO_FORTRAN_3D((*k_new[m_start])[mfi]), BL_TO_FORTRAN_3D((*k_new[m_end])[mfi]), BL_TO_FORTRAN_3D((*A_new[m_start])[mfi]), - BL_TO_FORTRAN_3D((*A_old[0])[mfi]), - BL_TO_FORTRAN_3D((*A_old[1])[mfi]), - BL_TO_FORTRAN_3D((*R_old[0])[mfi]), - BL_TO_FORTRAN_3D((*R_old[1])[mfi]), + BL_TO_FORTRAN_3D((*R_old[m_start])[mfi]), + BL_TO_FORTRAN_3D(C2), &sdc_iteration, &m_start); } else { diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index edb05331e8..618b6e9488 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1336,14 +1336,152 @@ subroutine ca_sdc_compute_initial_guess(lo, hi, & end subroutine ca_sdc_compute_initial_guess + subroutine ca_sdc_compute_C2_lobatto(lo, hi, dt_m, dt, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + R_0_old, R0lo, R0hi, & + R_1_old, R1lo, R1hi, & + C, Clo, Chi, & + m_start) bind(C, name="ca_sdc_compute_C2_lobatto") + ! compute the source term C for the 2nd order Lobatto update + + ! Here, dt_m is the timestep between time-nodes m and m+1 + + use meth_params_module, only : NVAR + use amrex_constants_module, only : ZERO, HALF + use burn_type_module, only : burn_t + use network, only : nspec, nspec_evolve + use react_util_module + + implicit none + + integer, intent(in) :: lo(3), hi(3) + real(rt), intent(in) :: dt_m, dt + integer, intent(in) :: Amlo(3), Amhi(3) + integer, intent(in) :: A0lo(3), A0hi(3) + integer, intent(in) :: A1lo(3), A1hi(3) + integer, intent(in) :: R0lo(3), R0hi(3) + integer, intent(in) :: R1lo(3), R1hi(3) + integer, intent(in) :: Clo(3), Chi(3) + integer, intent(in) :: m_start + + real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) + real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) + real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) + + real(rt), intent(in) :: R_0_old(R0lo(1):R0hi(1), R0lo(2):R0hi(2), R0lo(3):R0hi(3), NVAR) + real(rt), intent(in) :: R_1_old(R1lo(1):R1hi(1), R1lo(2):R1hi(2), R1lo(3):R1hi(3), NVAR) + + real(rt), intent(out) :: C(Clo(1):Chi(1), Clo(2):Chi(2), Clo(3):Chi(3), NVAR) + + integer :: i, j, k + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + ! construct the source term to the update for 2nd order + ! Lobatto, there is no advective correction, and we have + ! C = - R(U^{m+1,k}) + I_m^{m+1}/dt + C(i,j,k,:) = -R_1_old(i,j,k,:) + & + HALF * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + & + HALF * (R_0_old(i,j,k,:) + R_1_old(i,j,k,:)) + end do + end do + end do + + end subroutine ca_sdc_compute_C2_lobatto + + + subroutine ca_sdc_compute_C2_radau(lo, hi, dt_m, dt, & + A_m, Amlo, Amhi, & + A_0_old, A0lo, A0hi, & + A_1_old, A1lo, A1hi, & + A_2_old, A2lo, A2hi, & + R_0_old, R0lo, R0hi, & + R_1_old, R1lo, R1hi, & + R_2_old, R2lo, R2hi, & + C, Clo, Chi, & + m_start) bind(C, name="ca_sdc_compute_C2_radau") + ! compute the source term C for the 2nd order Radau update + + ! Here, dt_m is the timestep between time-nodes m and m+1 + + use meth_params_module, only : NVAR + use amrex_constants_module, only : ZERO, HALF, ONE, FIVE + use burn_type_module, only : burn_t + use network, only : nspec, nspec_evolve + use react_util_module + + implicit none + + integer, intent(in) :: lo(3), hi(3) + real(rt), intent(in) :: dt_m, dt + integer, intent(in) :: Amlo(3), Amhi(3) + integer, intent(in) :: A0lo(3), A0hi(3) + integer, intent(in) :: A1lo(3), A1hi(3) + integer, intent(in) :: A2lo(3), A2hi(3) + integer, intent(in) :: R0lo(3), R0hi(3) + integer, intent(in) :: R1lo(3), R1hi(3) + integer, intent(in) :: R2lo(3), R2hi(3) + integer, intent(in) :: Clo(3), Chi(3) + integer, intent(in) :: m_start + + real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) + real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) + real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) + real(rt), intent(in) :: A_2_old(A2lo(1):A2hi(1), A2lo(2):A2hi(2), A2lo(3):A2hi(3), NVAR) + + real(rt), intent(in) :: R_0_old(R0lo(1):R0hi(1), R0lo(2):R0hi(2), R0lo(3):R0hi(3), NVAR) + real(rt), intent(in) :: R_1_old(R1lo(1):R1hi(1), R1lo(2):R1hi(2), R1lo(3):R1hi(3), NVAR) + real(rt), intent(in) :: R_2_old(R2lo(1):R2hi(1), R2lo(2):R2hi(2), R2lo(3):R2hi(3), NVAR) + + real(rt), intent(out) :: C(Clo(1):Chi(1), Clo(2):Chi(2), Clo(3):Chi(3), NVAR) + + integer :: i, j, k + + ! construct the source term to the update for 2nd order + ! Radau + + if (m_start == 0) then + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + C(i,j,k,:) = -R_1_old(i,j,k,:) + & + (A_m(i,j,k,:) - A_0_old(i,j,k,:)) + & + (dt/dt_m) * (ONE/12.0_rt) * & + FIVE*(A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) - & + (A_2_old(i,j,k,:) + R_2_old(i,j,k,:)) + end do + end do + end do + + else if (m_start == 1) then + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + + C(i,j,k,:) = -R_2_old(i,j,k,:) + & + (A_m(i,j,k,:) - A_1_old(i,j,k,:)) + & + (dt/dt_m) * (ONE/3.0_rt) * & + (A_1_old(i,j,k,:) + R_1_old(i,j,k,:)) + & + (A_2_old(i,j,k,:) + R_2_old(i,j,k,:)) + end do + end do + end do + end if + + end subroutine ca_sdc_compute_C2_radau + + subroutine ca_sdc_update_o2(lo, hi, dt_m, & k_m, kmlo, kmhi, & k_n, knlo, knhi, & A_m, Amlo, Amhi, & - A_0_old, A0lo, A0hi, & - A_1_old, A1lo, A1hi, & - R_0_old, R0lo, R0hi, & - R_1_old, R1lo, R1hi, & + R_m_old, Rmlo, Rmhi, & + C, Clo, Chi, & sdc_iteration, & m_start) bind(C, name="ca_sdc_update_o2") ! update k_m to k_n via advection -- this is a second-order accurate update @@ -1363,10 +1501,8 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & integer, intent(in) :: kmlo(3), kmhi(3) integer, intent(in) :: knlo(3), knhi(3) integer, intent(in) :: Amlo(3), Amhi(3) - integer, intent(in) :: A0lo(3), A0hi(3) - integer, intent(in) :: A1lo(3), A1hi(3) - integer, intent(in) :: R0lo(3), R0hi(3) - integer, intent(in) :: R1lo(3), R1hi(3) + integer, intent(in) :: Rmlo(3), Rmhi(3) + integer, intent(in) :: Clo(3), Chi(3) integer, intent(in) :: sdc_iteration, m_start @@ -1374,32 +1510,21 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & real(rt), intent(inout) :: k_n(knlo(1):knhi(1), knlo(2):knhi(2), knlo(3):knhi(3), NVAR) real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) - real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) - real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) - - real(rt), intent(in) :: R_0_old(R0lo(1):R0hi(1), R0lo(2):R0hi(2), R0lo(3):R0hi(3), NVAR) - real(rt), intent(in) :: R_1_old(R1lo(1):R1hi(1), R1lo(2):R1hi(2), R1lo(3):R1hi(3), NVAR) + real(rt), intent(in) :: R_m_old(Rmlo(1):Rmhi(1), Rmlo(2):Rmhi(2), Rmlo(3):Rmhi(3), NVAR) + real(rt), intent(in) :: C(Clo(1):Chi(1), Clo(2):Chi(2), Clo(3):Chi(3), NVAR) integer :: i, j, k type(burn_t) :: burn_state - real(rt) :: U_old(NVAR), U_new(NVAR), C(NVAR), R_full(NVAR) + real(rt) :: U_old(NVAR), U_new(NVAR), R_full(NVAR), C_zone(NVAR) - - ! now consider the reacting system do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) U_old(:) = k_m(i,j,k,:) - - ! construct the source term to the update - ! for 2nd order, there is no advective correction, and we have - ! C = - R(U^{m+1,k}) + I_m^{m+1}/dt - C(:) = -R_1_old(i,j,k,:) + & - HALF * (A_0_old(i,j,k,:) + A_1_old(i,j,k,:)) + & - HALF * (R_0_old(i,j,k,:) + R_1_old(i,j,k,:)) + C_zone(:) = C(i,j,k,:) ! only burn if we are within the temperature and density ! limits for burning @@ -1415,12 +1540,12 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & ! the first iteration, let's try to extrapolate forward ! in time. if (sdc_iteration == 0) then - U_new(:) = U_old(:) + dt_m * A_m(i,j,k,:) + dt_m * R_0_old(i,j,k,:) + U_new(:) = U_old(:) + dt_m * A_m(i,j,k,:) + dt_m * R_m_old(i,j,k,:) else U_new(:) = k_n(i,j,k,:) endif - call sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) + call sdc_solve(dt_m, U_old, U_new, C_zone, sdc_iteration) ! we solved our system to some tolerance, but let's be sure we are conservative by ! reevaluating the reactions and then doing the full step update @@ -1428,7 +1553,7 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & end if - U_new(:) = U_old(:) + dt_m * R_full(:) + dt_m * C(:) + U_new(:) = U_old(:) + dt_m * R_full(:) + dt_m * C_zone(:) ! copy back to k_n k_n(i,j,k,:) = U_new(:) diff --git a/Source/hydro/Castro_hydro_F.H b/Source/hydro/Castro_hydro_F.H index 65f19998c2..bcc16859c0 100644 --- a/Source/hydro/Castro_hydro_F.H +++ b/Source/hydro/Castro_hydro_F.H @@ -552,16 +552,40 @@ extern "C" const int* m_start); #ifdef REACTIONS + void ca_sdc_compute_C2_lobatto + (const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(A_m), + const BL_FORT_FAB_ARG_3D(A_0_old), + const BL_FORT_FAB_ARG_3D(A_1_old), + const BL_FORT_FAB_ARG_3D(R_0_old), + const BL_FORT_FAB_ARG_3D(R_1_old), + BL_FORT_FAB_ARG_3D(C2), + const int* m_start); + + void ca_sdc_compute_C2_radau + (const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(A_m), + const BL_FORT_FAB_ARG_3D(A_0_old), + const BL_FORT_FAB_ARG_3D(A_1_old), + const BL_FORT_FAB_ARG_3D(A_2_old), + const BL_FORT_FAB_ARG_3D(R_0_old), + const BL_FORT_FAB_ARG_3D(R_1_old), + const BL_FORT_FAB_ARG_3D(R_2_old), + BL_FORT_FAB_ARG_3D(C2), + const int* m_start); + void ca_sdc_update_o2 (const int* lo, const int* hi, - const amrex::Real* time, + const amrex::Real* dt_m, const BL_FORT_FAB_ARG_3D(k_m), BL_FORT_FAB_ARG_3D(k_n), - BL_FORT_FAB_ARG_3D(A_m), - BL_FORT_FAB_ARG_3D(A_0_old), - BL_FORT_FAB_ARG_3D(A_1_old), - BL_FORT_FAB_ARG_3D(R_0_old), - BL_FORT_FAB_ARG_3D(R_1_old), + const BL_FORT_FAB_ARG_3D(A_m), + const BL_FORT_FAB_ARG_3D(R_m_old), + const BL_FORT_FAB_ARG_3D(C2), const int* sdc_iteration, const int* m_start); @@ -581,16 +605,33 @@ extern "C" const BL_FORT_FAB_ARG_3D(C), const BL_FORT_FAB_ARG_3D(R_new)); - void ca_sdc_compute_C4(const int* lo, const int* hi, - const BL_FORT_FAB_ARG_3D(A_m), - const BL_FORT_FAB_ARG_3D(A_0), - const BL_FORT_FAB_ARG_3D(A_1), - const BL_FORT_FAB_ARG_3D(A_2), - const BL_FORT_FAB_ARG_3D(R_0), - const BL_FORT_FAB_ARG_3D(R_1), - const BL_FORT_FAB_ARG_3D(R_2), - BL_FORT_FAB_ARG_3D(C), - const int* m_start); + void ca_sdc_compute_C4_lobatto(const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(A_m), + const BL_FORT_FAB_ARG_3D(A_0), + const BL_FORT_FAB_ARG_3D(A_1), + const BL_FORT_FAB_ARG_3D(A_2), + const BL_FORT_FAB_ARG_3D(R_0), + const BL_FORT_FAB_ARG_3D(R_1), + const BL_FORT_FAB_ARG_3D(R_2), + BL_FORT_FAB_ARG_3D(C), + const int* m_start); + + void ca_sdc_compute_C4_radau(const int* lo, const int* hi, + const amrex::Real* dt_m, + const amrex::Real* dt, + const BL_FORT_FAB_ARG_3D(A_m), + const BL_FORT_FAB_ARG_3D(A_0), + const BL_FORT_FAB_ARG_3D(A_1), + const BL_FORT_FAB_ARG_3D(A_2), + const BL_FORT_FAB_ARG_3D(A_3), + const BL_FORT_FAB_ARG_3D(R_0), + const BL_FORT_FAB_ARG_3D(R_1), + const BL_FORT_FAB_ARG_3D(R_2), + const BL_FORT_FAB_ARG_3D(R_3), + BL_FORT_FAB_ARG_3D(C), + const int* m_start); void ca_sdc_compute_initial_guess(const int* lo, const int* hi, const BL_FORT_FAB_ARG_3D(U_old), From 2420b38cf2724169a409c34e7dffc1fd2683228a Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 6 Sep 2019 06:04:17 -0400 Subject: [PATCH 42/72] fix interface --- Source/driver/sdc_util.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 618b6e9488..8232d1a772 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1120,6 +1120,7 @@ end subroutine ca_sdc_update_advection_o4_radau #ifdef REACTIONS subroutine ca_sdc_compute_C4_lobatto(lo, hi, & + dt_m, dt, & A_m, Amlo, Amhi, & A_0_old, A0lo, A0hi, & A_1_old, A1lo, A1hi, & @@ -1149,6 +1150,7 @@ subroutine ca_sdc_compute_C4_lobatto(lo, hi, & integer, intent(in) :: Clo(3), Chi(3) integer, intent(in) :: m_start + real(rt), intent(in) :: dt_m, dt real(rt), intent(in) :: A_m(Amlo(1):Amhi(1), Amlo(2):Amhi(2), Amlo(3):Amhi(3), NVAR) real(rt), intent(in) :: A_0_old(A0lo(1):A0hi(1), A0lo(2):A0hi(2), A0lo(3):A0hi(3), NVAR) real(rt), intent(in) :: A_1_old(A1lo(1):A1hi(1), A1lo(2):A1hi(2), A1lo(3):A1hi(3), NVAR) @@ -1194,7 +1196,8 @@ subroutine ca_sdc_compute_C4_lobatto(lo, hi, & end subroutine ca_sdc_compute_C4_lobatto - subroutine ca_sdc_compute_C4_radau(lo, hi, dt_m, dt, & + subroutine ca_sdc_compute_C4_radau(lo, hi, & + dt_m, dt, & A_m, Amlo, Amhi, & A_0_old, A0lo, A0hi, & A_1_old, A1lo, A1hi, & From aada5fa16a8ca6fab656f639fa0d749ce589cb41 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 13 Sep 2019 11:16:45 -0400 Subject: [PATCH 43/72] a test inputs file for SDC 2 --- Exec/science/Detonation/inputs-det-x.sdc2 | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 Exec/science/Detonation/inputs-det-x.sdc2 diff --git a/Exec/science/Detonation/inputs-det-x.sdc2 b/Exec/science/Detonation/inputs-det-x.sdc2 new file mode 100644 index 0000000000..1c5f606c74 --- /dev/null +++ b/Exec/science/Detonation/inputs-det-x.sdc2 @@ -0,0 +1,74 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 15000 +stop_time = 0.2 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 0 0 0 +geometry.coord_sys = 0 # 0 => cart, 1 => RZ 2=>spherical +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 4.e4 2500 2500 +amr.n_cell = 512 16 16 + + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 3 4 4 +castro.hi_bc = 2 4 4 + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 1 + +castro.time_integration_method = 2 +castro.sdc_order = 2 +castro.sdc_quadrature = 1 +#castro.sdc_extra = 1 + +castro.sdc_solver = 1 +castro.sdc_solver_tol_dens = 1.e-5 +castro.sdc_solver_tol_spec = 1.e-5 +castro.sdc_solver_tol_ener = 1.e-5 +castro.sdc_use_analytic_jac = 1 + +castro.ppm_type = 0 + +castro.use_flattening = 1 + +castro.small_temp = 1.e7 + +castro.riemann_solver = 0 + +# TIME STEP CONTROL +castro.cfl = 0.5 # cfl number for hyperbolic system +castro.init_shrink = 0.1 # scale back initial timestep +castro.change_max = 1.05 # scale back initial timestep +castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp +#amr.grid_log = grdlog # name of grid logging file + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +amr.ref_ratio = 2 2 2 2 # refinement ratio +amr.regrid_int = 2 2 2 2 # how often to regrid +amr.blocking_factor = 4 # block factor in grid generation +amr.max_grid_size = 64 +amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est + +# CHECKPOINT FILES +amr.check_file = det_x_chk # root name of checkpoint file +amr.check_int = 300 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = det_x_plt # root name of plotfile +amr.plot_per = 1.e-7 +amr.derive_plot_vars = ALL + +#PROBIN FILENAME +amr.probin_file = probin-det-x From 6a9a815442b67ce2cd29d08b63df21a9851cde1c Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 13 Sep 2019 11:16:59 -0400 Subject: [PATCH 44/72] more meaningful SDC progress messages --- Source/driver/Castro_advance_sdc.cpp | 3 +++ Source/hydro/Castro_mol_hydro.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/driver/Castro_advance_sdc.cpp b/Source/driver/Castro_advance_sdc.cpp index 7f4b84ceeb..d59aee8519 100644 --- a/Source/driver/Castro_advance_sdc.cpp +++ b/Source/driver/Castro_advance_sdc.cpp @@ -207,6 +207,9 @@ Castro::do_advance_sdc (Real time, // this if we are on the final node (since there is nothing to // update to if (m < SDC_NODES-1) { + + std::cout << "... doing the SDC update, iteration = " << sdc_iteration << " from node " << m << " to " << m+1 << std::endl; + do_sdc_update(m, m+1, dt); //(dt_sdc[m+1] - dt_sdc[m])*dt); // we now have a new value of k_new[m+1], do a clean_state on it diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index bef51eb067..2dd02624d6 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -24,7 +24,7 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Real strt_time = ParallelDescriptor::second(); if (verbose && ParallelDescriptor::IOProcessor()) { - std::cout << "... SDC iteration: " << sdc_iteration << "; current node: " << current_sdc_node << std::endl; + std::cout << "... construct advection term, SDC iteration: " << sdc_iteration << "; current node: " << current_sdc_node << std::endl; } From 84c195178db0a583c0873bdacb59889a01b7bc5b Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 13 Sep 2019 14:38:14 -0400 Subject: [PATCH 45/72] add some debugging output for when Newton fails --- Source/driver/sdc_util.F90 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 8232d1a772..5677cd8912 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -55,7 +55,7 @@ subroutine sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) ! reaction update. It either directly calls the Newton method or first ! tries VODE and then does the Newton update. - use meth_params_module, only : NVAR, sdc_solver + use meth_params_module, only : NVAR, sdc_solver, URHO, UTEMP, UEINT, UFS use amrex_constants_module, only : ZERO, HALF, ONE use burn_type_module, only : burn_t use react_util_module @@ -72,6 +72,8 @@ subroutine sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) integer :: ierr + ! for debugging + real(rt) :: U_orig(NVAR) if (sdc_solver == NEWTON_SOLVE) then ! we are going to assume we already have a good guess for the @@ -81,6 +83,13 @@ subroutine sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) ! failing? if (ierr /= NEWTON_SUCCESS) then + print *, "Newton convergence failure" + print *, " input state:" + print *, " density: ", U_orig(URHO) + print *, " temperature : ", U_orig(UTEMP) + print *, " (rho e): ", U_orig(UEINT) + print *, " mass fractions: ", U_orig(UFS:UFS-1+nspec)/U_orig(URHO) + print *, " " call castro_error("Newton subcycling failed in sdc_solve") end if From 9b22c1f8c971e48c1192ad81b79022300f95f76b Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 13 Sep 2019 16:08:55 -0400 Subject: [PATCH 46/72] remove unused function + add species normalization to f_sdc_jac --- Source/driver/sdc_util.F90 | 87 ++++++++------------------------------ 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 5677cd8912..a668478b53 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -158,6 +158,7 @@ subroutine sdc_newton_subdivide(dt_m, U_old, U_new, C, sdc_iteration, ierr) do n = 1, nspec U_begin(UFS-1+n) = max(ZERO, U_begin(UFS-1+n)) end do + sum_rhoX = sum(U_begin(UFS:UFS-1+nspec)) U_begin(UFS:UFS-1+nspec) = U_begin(UFS:UFS-1+nspec) * U_begin(URHO)/sum_rhoX @@ -650,70 +651,7 @@ subroutine jac_ode(n, time, U, ml, mu, Jac, nrowpd, rpar, ipar) end subroutine jac_ode - subroutine f_sdc(n, U, f, iflag, rpar) - ! this is used by the Newton solve to compute the Jacobian via differencing - - use rpar_sdc_module - use meth_params_module, only : nvar, URHO, UFS, UEDEN, UMX, UMZ, UEINT, UTEMP, sdc_solve_for_rhoe - use network, only : nspec, nspec_evolve - use burn_type_module - use react_util_module - - ! this computes the function we need to zero for the SDC update - implicit none - - integer,intent(in) :: n - real(rt), intent(in) :: U(0:n-1) - real(rt), intent(out) :: f(0:n-1) - integer, intent(inout) :: iflag !! leave this untouched - real(rt), intent(inout) :: rpar(0:n_rpar-1) - - real(rt) :: U_full(nvar), R_full(nvar) - real(rt) :: R_react(0:n-1), f_source(0:n-1) - type(burn_t) :: burn_state - - real(rt) :: dt_m - - ! we are not solving the momentum equations - ! create a full state -- we need this for some interfaces - U_full(URHO) = U(0) - U_full(UFS:UFS-1+nspec_evolve) = U(1:nspec_evolve) - if (sdc_solve_for_rhoe == 1) then - U_full(UEINT) = U(nspec_evolve+1) - U_full(UEDEN) = rpar(irp_evar) - else - U_full(UEDEN) = U(nspec_evolve+1) - U_full(UEINT) = rpar(irp_evar) - endif - - U_full(UMX:UMZ) = rpar(irp_mom:irp_mom+2) - U_full(UFS+nspec_evolve:UFS-1+nspec) = rpar(irp_spec:irp_spec-1+(nspec-nspec_evolve)) - - ! unpack rpar - dt_m = rpar(irp_dt) - f_source(:) = rpar(irp_f_source:irp_f_source-1+nspec_evolve+2) - - ! initial guess for T - U_full(UTEMP) = rpar(irp_temp) - - call single_zone_react_source(U_full, R_full, 0,0,0, burn_state) - - ! update guess for next time - rpar(irp_temp) = U_full(UTEMP) - - R_react(0) = R_full(URHO) - R_react(1:nspec_evolve) = R_full(UFS:UFS-1+nspec_evolve) - if (sdc_solve_for_rhoe == 1) then - R_react(nspec_evolve+1) = R_full(UEINT) - else - R_react(nspec_evolve+1) = R_full(UEDEN) - endif - - f(:) = U(:) - dt_m * R_react(:) - f_source(:) - - end subroutine f_sdc - - subroutine f_sdc_jac(n, U, f, Jac, ldjac, iflag, rpar) + subroutine f_sdc_jac(neq, U, f, Jac, ldjac, iflag, rpar) ! this is used with the Newton solve and returns f and the Jacobian use rpar_sdc_module @@ -729,22 +667,23 @@ subroutine f_sdc_jac(n, U, f, Jac, ldjac, iflag, rpar) ! this computes the function we need to zero for the SDC update implicit none - integer,intent(in) :: n, ldjac - real(rt), intent(in) :: U(0:n-1) - real(rt), intent(out) :: f(0:n-1) - real(rt), intent(out) :: Jac(0:ldjac-1,0:n-1) + integer,intent(in) :: neq, ldjac + real(rt), intent(in) :: U(0:neq-1) + real(rt), intent(out) :: f(0:neq-1) + real(rt), intent(out) :: Jac(0:ldjac-1,0:neq-1) integer, intent(inout) :: iflag !! leave this untouched real(rt), intent(inout) :: rpar(0:n_rpar-1) real(rt) :: U_full(nvar), R_full(nvar) - real(rt) :: R_react(0:n-1), f_source(0:n-1) + real(rt) :: R_react(0:neq-1), f_source(0:neq-1) type(burn_t) :: burn_state type(eos_t) :: eos_state real(rt) :: dt_m real(rt) :: denom real(rt) :: dRdw(0:nspec_evolve+1, 0:nspec_evolve+1), dwdU(0:nspec_evolve+1, 0:nspec_evolve+1) - integer :: m + integer :: m, k + real(rt) :: sum_rhoX ! we are not solving the momentum equations ! create a full state -- we need this for some interfaces @@ -761,6 +700,14 @@ subroutine f_sdc_jac(n, U, f, Jac, ldjac, iflag, rpar) U_full(UMX:UMZ) = rpar(irp_mom:irp_mom+2) U_full(UFS+nspec_evolve:UFS-1+nspec) = rpar(irp_spec:irp_spec-1+(nspec-nspec_evolve)) + ! normalize the species + do k = 1, nspec + U_full(UFS-1+k) = max(ZERO, U_full(UFS-1+k)) + end do + + sum_rhoX = sum(U_full(UFS:UFS-1+nspec)) + U_full(UFS:UFS-1+nspec) = U_full(UFS:UFS-1+nspec) * U_full(URHO)/sum_rhoX + ! unpack rpar dt_m = rpar(irp_dt) f_source(:) = rpar(irp_f_source:irp_f_source-1+nspec_evolve+2) From 8c2a46b915be5c53568a5e524f50de0b947543a6 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sat, 14 Sep 2019 12:01:25 -0400 Subject: [PATCH 47/72] more normalization work --- Source/driver/sdc_util.F90 | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index a668478b53..57a83c6892 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -151,8 +151,6 @@ subroutine sdc_newton_subdivide(dt_m, U_old, U_new, C, sdc_iteration, ierr) do while (nsub < MAX_NSUB .and. ierr /= NEWTON_SUCCESS) dt_sub = dt_m / nsub do isub = 1, nsub - call sdc_newton_solve(dt_sub, U_begin, U_new, C, sdc_iteration, ierr) - U_begin(:) = U_new(:) ! normalize species do n = 1, nspec @@ -162,6 +160,9 @@ subroutine sdc_newton_subdivide(dt_m, U_old, U_new, C, sdc_iteration, ierr) sum_rhoX = sum(U_begin(UFS:UFS-1+nspec)) U_begin(UFS:UFS-1+nspec) = U_begin(UFS:UFS-1+nspec) * U_begin(URHO)/sum_rhoX + call sdc_newton_solve(dt_sub, U_begin, U_new, C, sdc_iteration, ierr) + U_begin(:) = U_new(:) + end do nsub = nsub * 2 end do @@ -225,6 +226,9 @@ subroutine sdc_newton_solve(dt_m, U_old, U_new, C, sdc_iteration, ierr) integer :: max_newton_iter + real(rt) :: xn(nspec) + integer :: k + ierr = NEWTON_SUCCESS ! the tolerance we are solving to may depend on the iteration @@ -317,6 +321,18 @@ subroutine sdc_newton_solve(dt_m, U_old, U_new, C, sdc_iteration, ierr) U_react(:) = U_react(:) + dU_react(:) + ! we still need to normalize here + xn(1:nspec_evolve) = U_react(1:nspec_evolve)/U_react(0) + xn(nspec_evolve+1:nspec) = rpar(irp_spec:irp_spec-1+(nspec-nspec_evolve))/U_react(0) + + do k = 1, nspec + xn(k) = max(ZERO, xn(k)) + end do + xn(:) = xn(:)/sum(xn) + + U_react(1:nspec_evolve) = U_react(0) * xn(1:nspec_evolve) + rpar(irp_spec:irp_spec-1+(nspec-nspec_evolve)) = U_react(0) * xn(nspec_evolve+1:nspec) + eps_tot(0) = tol_dens * abs(U_react(0)) + sdc_solver_atol ! for species, atol is the mass fraction limit, so we multiply by density to get a partial density limit eps_tot(1:nspec_evolve) = tol_spec * abs(U_react(1:nspec_evolve)) + sdc_solver_atol * abs(U_react(0)) @@ -651,6 +667,7 @@ subroutine jac_ode(n, time, U, ml, mu, Jac, nrowpd, rpar, ipar) end subroutine jac_ode + subroutine f_sdc_jac(neq, U, f, Jac, ldjac, iflag, rpar) ! this is used with the Newton solve and returns f and the Jacobian From 82d8c94310c77a609a4f0b387bb14b785bb17962 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sat, 14 Sep 2019 20:35:47 -0400 Subject: [PATCH 48/72] fix a comment --- Source/driver/Castro_advance_sdc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/driver/Castro_advance_sdc.cpp b/Source/driver/Castro_advance_sdc.cpp index d59aee8519..146d205fd4 100644 --- a/Source/driver/Castro_advance_sdc.cpp +++ b/Source/driver/Castro_advance_sdc.cpp @@ -77,10 +77,10 @@ Castro::do_advance_sdc (Real time, // the next chunk of code constructs the advective term for the - // current node, m. First we get the sources, then convert to - // primitive, then get the source term from the MOL driver. Note, - // for m = 0, we only have to do all of this the first iteration, - // since that state never changes + // current node, m. First we get the sources, then full hydro + // source term from the MOL driver. Note, for m = 0, we only have + // to do all of this the first iteration, since that state never + // changes if (!(sdc_iteration > 0 && m == 0) && !(sdc_iteration == sdc_order+sdc_extra-1 && m == SDC_NODES-1)) { From d0938d0dd133b4bdf1c5a2cd538088a2adca0eb1 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 15 Sep 2019 10:22:07 -0400 Subject: [PATCH 49/72] use small_x in the normalization of species --- Source/driver/sdc_util.F90 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 57a83c6892..3f7fc92de6 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -126,6 +126,7 @@ subroutine sdc_newton_subdivide(dt_m, U_old, U_new, C, sdc_iteration, ierr) use amrex_constants_module, only : ZERO, HALF, ONE use network, only : nspec, nspec_evolve use rpar_sdc_module + use extern_probin_module, only : small_x implicit none @@ -154,7 +155,7 @@ subroutine sdc_newton_subdivide(dt_m, U_old, U_new, C, sdc_iteration, ierr) ! normalize species do n = 1, nspec - U_begin(UFS-1+n) = max(ZERO, U_begin(UFS-1+n)) + U_begin(UFS-1+n) = max(small_x, U_begin(UFS-1+n)) end do sum_rhoX = sum(U_begin(UFS:UFS-1+nspec)) @@ -187,6 +188,7 @@ subroutine sdc_newton_solve(dt_m, U_old, U_new, C, sdc_iteration, ierr) use react_util_module use network, only : nspec, nspec_evolve use rpar_sdc_module + use extern_probin_module, only : small_x implicit none @@ -326,7 +328,7 @@ subroutine sdc_newton_solve(dt_m, U_old, U_new, C, sdc_iteration, ierr) xn(nspec_evolve+1:nspec) = rpar(irp_spec:irp_spec-1+(nspec-nspec_evolve))/U_react(0) do k = 1, nspec - xn(k) = max(ZERO, xn(k)) + xn(k) = max(small_x, xn(k)) end do xn(:) = xn(:)/sum(xn) @@ -593,6 +595,7 @@ subroutine jac_ode(n, time, U, ml, mu, Jac, nrowpd, rpar, ipar) use eos_module, only : eos use rpar_sdc_module use react_util_module + use extern_probin_module, only : small_x implicit none @@ -680,6 +683,7 @@ subroutine f_sdc_jac(neq, U, f, Jac, ldjac, iflag, rpar) use eos_type_module, only : eos_t, eos_input_re use eos_module, only : eos use amrex_constants_module, only : ZERO, HALF, ONE + use extern_probin_module, only : small_x ! this computes the function we need to zero for the SDC update implicit none @@ -719,7 +723,7 @@ subroutine f_sdc_jac(neq, U, f, Jac, ldjac, iflag, rpar) ! normalize the species do k = 1, nspec - U_full(UFS-1+k) = max(ZERO, U_full(UFS-1+k)) + U_full(UFS-1+k) = max(small_x, U_full(UFS-1+k)) end do sum_rhoX = sum(U_full(UFS:UFS-1+nspec)) From 4bfb92d6dd9dadd2282118e97c63792c71d9a8d7 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 15 Sep 2019 10:24:34 -0400 Subject: [PATCH 50/72] sync up a bit with the regression version --- Exec/science/Detonation/inputs-det-x.sdc2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Exec/science/Detonation/inputs-det-x.sdc2 b/Exec/science/Detonation/inputs-det-x.sdc2 index 1c5f606c74..286674d7c4 100644 --- a/Exec/science/Detonation/inputs-det-x.sdc2 +++ b/Exec/science/Detonation/inputs-det-x.sdc2 @@ -24,9 +24,11 @@ castro.do_react = 1 castro.time_integration_method = 2 castro.sdc_order = 2 -castro.sdc_quadrature = 1 +castro.sdc_quadrature = 0 #castro.sdc_extra = 1 +castro.sdc_solve_for_rhoe = 1 + castro.sdc_solver = 1 castro.sdc_solver_tol_dens = 1.e-5 castro.sdc_solver_tol_spec = 1.e-5 @@ -46,6 +48,7 @@ castro.cfl = 0.5 # cfl number for hyperbolic system castro.init_shrink = 0.1 # scale back initial timestep castro.change_max = 1.05 # scale back initial timestep castro.dt_cutoff = 5.e-20 # level 0 timestep below which we halt +castro.dtnuc_e = 0.25 # DIAGNOSTICS & VERBOSITY castro.sum_interval = 1 # timesteps between computing mass From 4a2b4b82409cc6954728cf5ec15913f588191fc5 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 15 Sep 2019 18:21:30 -0400 Subject: [PATCH 51/72] fix merge issue --- CHANGES | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGES b/CHANGES index 4400c5be01..866aed4c3c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,3 @@ -<<<<<<< HEAD changes since the last release -- The SDC algorithm now implements the burning conditionals @@ -19,12 +18,9 @@ changes since the last release 1 = piecewise constant, 2 = 2nd order MC limiter, and >= 3 for 4th order MC limiter (previously there was no way to select the 2nd order limiter). -======= -changes since the last release: -- The Runge-Kutta based method-of-lines integration method has been removed in favor of the SDC integration. ->>>>>>> development # 19.09 From 8bdff09b288defdc545d1fad7542c51e09fab1b0 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 16 Sep 2019 17:21:59 -0400 Subject: [PATCH 52/72] fix names --- Exec/science/Detonation/sdc_tests/setup_runs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/setup_runs.sh b/Exec/science/Detonation/sdc_tests/setup_runs.sh index ebf594ae90..1fcfe3bdf5 100755 --- a/Exec/science/Detonation/sdc_tests/setup_runs.sh +++ b/Exec/science/Detonation/sdc_tests/setup_runs.sh @@ -1,4 +1,4 @@ -CASTRO_EXEC=./Castro1d.gnu.MPI.ex +CASTRO_EXEC=./Castro1d.gnu.ex NEEDED_FILES=" ${CASTRO_EXEC} @@ -40,7 +40,7 @@ do for nz in ${NZONES} do - rdir=flame_z${nz}_c${c}_lobatto_sdc4 + rdir=det_z${nz}_c${c}_lobatto_sdc4 if [ ! -d ${rdir} ]; then mkdir ${rdir} fi @@ -80,7 +80,7 @@ do for nz in ${NZONES} do - rdir=flame_z${nz}_c${c}_radau_sdc4 + rdir=det_z${nz}_c${c}_radau_sdc4 if [ ! -d ${rdir} ]; then mkdir ${rdir} fi @@ -114,7 +114,7 @@ do for nz in ${NZONES} do - rdir=flame_z${nz}_c${c}_strang_ctu + rdir=det_z${nz}_c${c}_strang_ctu if [ ! -d ${rdir} ]; then mkdir ${rdir} fi From 279c3ebfb01d4c4c0a1c765a361faf1ec028ab4c Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 16 Sep 2019 17:24:06 -0400 Subject: [PATCH 53/72] remove mpi --- Exec/science/Detonation/sdc_tests/setup_runs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/setup_runs.sh b/Exec/science/Detonation/sdc_tests/setup_runs.sh index 1fcfe3bdf5..d8b01a8720 100755 --- a/Exec/science/Detonation/sdc_tests/setup_runs.sh +++ b/Exec/science/Detonation/sdc_tests/setup_runs.sh @@ -53,7 +53,7 @@ do fi done - nohup mpiexec -n 4 ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & + nohup ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & cd .. done done @@ -93,7 +93,7 @@ do fi done - nohup mpiexec -n 4 ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & + nohup ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & cd .. done done @@ -127,7 +127,7 @@ do fi done - nohup mpiexec -n 4 ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & + nohup ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & cd .. done done From c2cab6090572fa0b4a1d6e28086a0634c1fa0c2c Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 16 Sep 2019 17:32:58 -0400 Subject: [PATCH 54/72] fix number of steps --- Exec/science/Detonation/sdc_tests/inputs.1d.sdc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/inputs.1d.sdc b/Exec/science/Detonation/sdc_tests/inputs.1d.sdc index 95f65ee5a1..cef4192c7e 100644 --- a/Exec/science/Detonation/sdc_tests/inputs.1d.sdc +++ b/Exec/science/Detonation/sdc_tests/inputs.1d.sdc @@ -1,5 +1,5 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- -max_step = 40 +max_step = 40000 stop_time = 0.2 # PROBLEM SIZE & GEOMETRY @@ -63,7 +63,7 @@ amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est # CHECKPOINT FILES amr.check_file = det_x_chk # root name of checkpoint file -amr.check_int = 300 # number of timesteps between checkpoints +amr.check_int = 1000 # number of timesteps between checkpoints # PLOTFILES amr.plot_file = det_x_plt # root name of plotfile From eb795386c89decf9bc66555b7c8cb0c4566cfe02 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 16 Sep 2019 20:08:11 -0400 Subject: [PATCH 55/72] fix quadrature values --- Exec/science/Detonation/sdc_tests/setup_runs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/setup_runs.sh b/Exec/science/Detonation/sdc_tests/setup_runs.sh index d8b01a8720..69ff10e3b2 100755 --- a/Exec/science/Detonation/sdc_tests/setup_runs.sh +++ b/Exec/science/Detonation/sdc_tests/setup_runs.sh @@ -26,7 +26,7 @@ stop_time=3.e-6" RUNPARAMS=" castro.time_integration_method=2 castro.sdc_order=4 -castro.sdc_quadrature=1 +castro.sdc_quadrature=0 castro.limit_fourth_order=1 castro.use_reconstructed_gamma1=1 castro.sdc_solve_for_rhoe=1 @@ -66,7 +66,7 @@ done RUNPARAMS=" castro.time_integration_method=2 castro.sdc_order=4 -castro.sdc_quadrature=2 +castro.sdc_quadrature=1 castro.limit_fourth_order=1 castro.use_reconstructed_gamma1=1 castro.sdc_solve_for_rhoe=1 From 631e542577b57bccb707eb8778955b1716091097 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 16 Sep 2019 20:08:30 -0400 Subject: [PATCH 56/72] check quadrature value --- Source/driver/Castro_advance.cpp | 1 + Source/driver/Castro_setup.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Source/driver/Castro_advance.cpp b/Source/driver/Castro_advance.cpp index b8ce22aff4..4f23c9eefd 100644 --- a/Source/driver/Castro_advance.cpp +++ b/Source/driver/Castro_advance.cpp @@ -540,6 +540,7 @@ Castro::initialize_advance(Real time, Real dt, int amr_iteration, int amr_ncycle MultiFab& S_old = get_old_data(State_Type); k_new.resize(SDC_NODES); + k_new[0].reset(new MultiFab(S_old, amrex::make_alias, 0, NUM_STATE)); for (int n = 1; n < SDC_NODES; ++n) { k_new[n].reset(new MultiFab(grids, dmap, NUM_STATE, 0)); diff --git a/Source/driver/Castro_setup.cpp b/Source/driver/Castro_setup.cpp index de79a234d6..6ee87c16b0 100644 --- a/Source/driver/Castro_setup.cpp +++ b/Source/driver/Castro_setup.cpp @@ -1143,5 +1143,7 @@ Castro::variableSetUp () amrex::Error("invalid value of sdc_order"); } + } else { + amrex::Error("invalid value of sdc_quadrature"); } } From cf6aeeea4d7c2db7272774481466ba535a4bedaf Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 17 Sep 2019 18:15:07 -0400 Subject: [PATCH 57/72] fix merge conflicts --- CHANGES | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/CHANGES b/CHANGES index c647532f66..866aed4c3c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,3 @@ -<<<<<<< HEAD -======= -<<<<<<< HEAD ->>>>>>> xrb_sdc changes since the last release -- The SDC algorithm now implements the burning conditionals @@ -22,18 +18,9 @@ changes since the last release 1 = piecewise constant, 2 = 2nd order MC limiter, and >= 3 for 4th order MC limiter (previously there was no way to select the 2nd order limiter). -<<<<<<< HEAD -- The Runge-Kutta based method-of-lines integration method has been removed in favor of the SDC integration. -======= -======= -changes since the last release: - - -- The Runge-Kutta based method-of-lines integration method has - been removed in favor of the SDC integration. ->>>>>>> development ->>>>>>> xrb_sdc # 19.09 From a43e53e7bd5120c4a60e324944938f39248ffbbf Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 17 Sep 2019 18:22:48 -0400 Subject: [PATCH 58/72] use amrex::Print() --- Source/driver/Castro_advance_sdc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/driver/Castro_advance_sdc.cpp b/Source/driver/Castro_advance_sdc.cpp index 146d205fd4..08c413099e 100644 --- a/Source/driver/Castro_advance_sdc.cpp +++ b/Source/driver/Castro_advance_sdc.cpp @@ -208,7 +208,7 @@ Castro::do_advance_sdc (Real time, // update to if (m < SDC_NODES-1) { - std::cout << "... doing the SDC update, iteration = " << sdc_iteration << " from node " << m << " to " << m+1 << std::endl; + amrex::Print() << "... doing the SDC update, iteration = " << sdc_iteration << " from node " << m << " to " << m+1 << std::endl; do_sdc_update(m, m+1, dt); //(dt_sdc[m+1] - dt_sdc[m])*dt); From 01cbb104311ffbfbcbe67673d6f54832f5a6bb11 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 18 Sep 2019 08:15:22 -0400 Subject: [PATCH 59/72] revert the overloading of plm_iorder now we have a new option plm_limiter that can choose the limiter if we are doing linear slopes --- Source/driver/_cpp_parameters | 8 ++- Source/driver/meth_params.F90 | 58 +++++++++++-------- .../driver/param_includes/castro_defaults.H | 3 +- .../param_includes/castro_job_info_tests.H | 3 +- Source/driver/param_includes/castro_params.H | 1 + Source/driver/param_includes/castro_queries.H | 1 + Source/hydro/slope_nd.F90 | 11 ++-- 7 files changed, 52 insertions(+), 33 deletions(-) diff --git a/Source/driver/_cpp_parameters b/Source/driver/_cpp_parameters index 2183731403..313e6e42eb 100644 --- a/Source/driver/_cpp_parameters +++ b/Source/driver/_cpp_parameters @@ -104,8 +104,12 @@ ppm_predict_gammae int 0 y ppm_reference_eigenvectors int 0 y # for piecewise linear, reconstruction order to use -# 1 = piecewise constant, 2 = MC limiter, >2 = 4th order MC limiter -plm_iorder int 3 y +# 1 = piecewise constant, 2 = piecewise linear +plm_iorder int 2 y + +# for piecewise linear, what limiter to use? +# 1 = 2nd order MC, 2 = 4th order MC +plm_limiter int 2 y # for piecewise linear, MOL or SDC, do well-balanced reconstruction on # pressure diff --git a/Source/driver/meth_params.F90 b/Source/driver/meth_params.F90 index 80fc77fe9b..c572e85f39 100644 --- a/Source/driver/meth_params.F90 +++ b/Source/driver/meth_params.F90 @@ -132,6 +132,7 @@ module meth_params_module integer, allocatable, save :: ppm_predict_gammae integer, allocatable, save :: ppm_reference_eigenvectors integer, allocatable, save :: plm_iorder + integer, allocatable, save :: plm_limiter integer, allocatable, save :: plm_well_balanced integer, allocatable, save :: hybrid_riemann integer, allocatable, save :: riemann_solver @@ -225,6 +226,7 @@ module meth_params_module attributes(managed) :: ppm_predict_gammae attributes(managed) :: ppm_reference_eigenvectors attributes(managed) :: plm_iorder +attributes(managed) :: plm_limiter attributes(managed) :: plm_well_balanced attributes(managed) :: hybrid_riemann attributes(managed) :: riemann_solver @@ -351,6 +353,7 @@ module meth_params_module !$acc create(ppm_predict_gammae) & !$acc create(ppm_reference_eigenvectors) & !$acc create(plm_iorder) & + !$acc create(plm_limiter) & !$acc create(plm_well_balanced) & !$acc create(hybrid_riemann) & !$acc create(riemann_solver) & @@ -557,7 +560,9 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para allocate(ppm_reference_eigenvectors) ppm_reference_eigenvectors = 0; allocate(plm_iorder) - plm_iorder = 3; + plm_iorder = 2; + allocate(plm_limiter) + plm_limiter = 2; allocate(plm_well_balanced) plm_well_balanced = 0; allocate(hybrid_riemann) @@ -714,6 +719,7 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para call pp%query("ppm_predict_gammae", ppm_predict_gammae) call pp%query("ppm_reference_eigenvectors", ppm_reference_eigenvectors) call pp%query("plm_iorder", plm_iorder) + call pp%query("plm_limiter", plm_limiter) call pp%query("plm_well_balanced", plm_well_balanced) call pp%query("hybrid_riemann", hybrid_riemann) call pp%query("riemann_solver", riemann_solver) @@ -786,30 +792,29 @@ subroutine ca_set_castro_method_params() bind(C, name="ca_set_castro_method_para !$acc device(time_integration_method, limit_fourth_order, use_reconstructed_gamma1) & !$acc device(hybrid_hydro, ppm_type, ppm_temp_fix) & !$acc device(ppm_predict_gammae, ppm_reference_eigenvectors, plm_iorder) & - !$acc device(plm_well_balanced, hybrid_riemann, riemann_solver) & - !$acc device(cg_maxiter, cg_tol, cg_blend) & - !$acc device(use_eos_in_riemann, riemann_speed_limit, use_flattening) & - !$acc device(transverse_use_eos, transverse_reset_density, transverse_reset_rhoe) & - !$acc device(dual_energy_eta1, dual_energy_eta2, use_pslope) & - !$acc device(limit_fluxes_on_small_dens, density_reset_method, allow_small_energy) & - !$acc device(do_sponge, sponge_implicit, first_order_hydro) & - !$acc device(hse_zero_vels, hse_interp_temp, hse_reflect_vels) & - !$acc device(sdc_order, sdc_extra, sdc_solver) & - !$acc device(sdc_solver_tol_dens, sdc_solver_tol_spec, sdc_solver_tol_ener) & - !$acc device(sdc_solver_atol, sdc_solver_relax_factor, sdc_solve_for_rhoe) & - !$acc device(sdc_use_analytic_jac, cfl, dtnuc_e) & - !$acc device(dtnuc_X, dtnuc_X_threshold, do_react) & - !$acc device(react_T_min, react_T_max, react_rho_min) & - !$acc device(react_rho_max, disable_shock_burning, T_guess) & - !$acc device(diffuse_temp, diffuse_cutoff_density, diffuse_cutoff_density_hi) & - !$acc device(diffuse_cond_scale_fac, do_grav, grav_source_type) & - !$acc device(do_rotation, rot_period, rot_period_dot) & - !$acc device(rotation_include_centrifugal, rotation_include_coriolis, rotation_include_domegadt) & - !$acc device(state_in_rotating_frame, rot_source_type, implicit_rotation_update) & - !$acc device(rot_axis, use_point_mass, point_mass) & - !$acc device(point_mass_fix_solution, do_acc, grown_factor) & - !$acc device(track_grid_losses, const_grav) & - !$acc device(get_g_from_phi) + !$acc device(plm_limiter, plm_well_balanced, hybrid_riemann) & + !$acc device(riemann_solver, cg_maxiter, cg_tol) & + !$acc device(cg_blend, use_eos_in_riemann, riemann_speed_limit) & + !$acc device(use_flattening, transverse_use_eos, transverse_reset_density) & + !$acc device(transverse_reset_rhoe, dual_energy_eta1, dual_energy_eta2) & + !$acc device(use_pslope, limit_fluxes_on_small_dens, density_reset_method) & + !$acc device(allow_small_energy, do_sponge, sponge_implicit) & + !$acc device(first_order_hydro, hse_zero_vels, hse_interp_temp) & + !$acc device(hse_reflect_vels, sdc_order, sdc_extra) & + !$acc device(sdc_solver, sdc_solver_tol_dens, sdc_solver_tol_spec) & + !$acc device(sdc_solver_tol_ener, sdc_solver_atol, sdc_solver_relax_factor) & + !$acc device(sdc_solve_for_rhoe, sdc_use_analytic_jac, cfl) & + !$acc device(dtnuc_e, dtnuc_X, dtnuc_X_threshold) & + !$acc device(do_react, react_T_min, react_T_max) & + !$acc device(react_rho_min, react_rho_max, disable_shock_burning) & + !$acc device(T_guess, diffuse_temp, diffuse_cutoff_density) & + !$acc device(diffuse_cutoff_density_hi, diffuse_cond_scale_fac, do_grav) & + !$acc device(grav_source_type, do_rotation, rot_period) & + !$acc device(rot_period_dot, rotation_include_centrifugal, rotation_include_coriolis) & + !$acc device(rotation_include_domegadt, state_in_rotating_frame, rot_source_type) & + !$acc device(implicit_rotation_update, rot_axis, use_point_mass) & + !$acc device(point_mass, point_mass_fix_solution, do_acc) & + !$acc device(grown_factor, track_grid_losses, const_grav, get_g_from_phi) #ifdef GRAVITY @@ -954,6 +959,9 @@ subroutine ca_finalize_meth_params() bind(C, name="ca_finalize_meth_params") if (allocated(plm_iorder)) then deallocate(plm_iorder) end if + if (allocated(plm_limiter)) then + deallocate(plm_limiter) + end if if (allocated(plm_well_balanced)) then deallocate(plm_well_balanced) end if diff --git a/Source/driver/param_includes/castro_defaults.H b/Source/driver/param_includes/castro_defaults.H index 40658c201a..0053bacbb4 100644 --- a/Source/driver/param_includes/castro_defaults.H +++ b/Source/driver/param_includes/castro_defaults.H @@ -45,7 +45,8 @@ int Castro::ppm_type = 1; int Castro::ppm_temp_fix = 0; int Castro::ppm_predict_gammae = 0; int Castro::ppm_reference_eigenvectors = 0; -int Castro::plm_iorder = 3; +int Castro::plm_iorder = 2; +int Castro::plm_limiter = 2; int Castro::plm_well_balanced = 0; int Castro::hybrid_riemann = 0; int Castro::riemann_solver = 0; diff --git a/Source/driver/param_includes/castro_job_info_tests.H b/Source/driver/param_includes/castro_job_info_tests.H index 242cc4160b..5330a4aa92 100644 --- a/Source/driver/param_includes/castro_job_info_tests.H +++ b/Source/driver/param_includes/castro_job_info_tests.H @@ -40,7 +40,8 @@ jobInfoFile << (Castro::ppm_type == 1 ? " " : "[*] ") << "castro.ppm_type = " jobInfoFile << (Castro::ppm_temp_fix == 0 ? " " : "[*] ") << "castro.ppm_temp_fix = " << Castro::ppm_temp_fix << std::endl; jobInfoFile << (Castro::ppm_predict_gammae == 0 ? " " : "[*] ") << "castro.ppm_predict_gammae = " << Castro::ppm_predict_gammae << std::endl; jobInfoFile << (Castro::ppm_reference_eigenvectors == 0 ? " " : "[*] ") << "castro.ppm_reference_eigenvectors = " << Castro::ppm_reference_eigenvectors << std::endl; -jobInfoFile << (Castro::plm_iorder == 3 ? " " : "[*] ") << "castro.plm_iorder = " << Castro::plm_iorder << std::endl; +jobInfoFile << (Castro::plm_iorder == 2 ? " " : "[*] ") << "castro.plm_iorder = " << Castro::plm_iorder << std::endl; +jobInfoFile << (Castro::plm_limiter == 2 ? " " : "[*] ") << "castro.plm_limiter = " << Castro::plm_limiter << std::endl; jobInfoFile << (Castro::plm_well_balanced == 0 ? " " : "[*] ") << "castro.plm_well_balanced = " << Castro::plm_well_balanced << std::endl; jobInfoFile << (Castro::hybrid_riemann == 0 ? " " : "[*] ") << "castro.hybrid_riemann = " << Castro::hybrid_riemann << std::endl; jobInfoFile << (Castro::riemann_solver == 0 ? " " : "[*] ") << "castro.riemann_solver = " << Castro::riemann_solver << std::endl; diff --git a/Source/driver/param_includes/castro_params.H b/Source/driver/param_includes/castro_params.H index 17da1e8d3b..80a0047a4a 100644 --- a/Source/driver/param_includes/castro_params.H +++ b/Source/driver/param_includes/castro_params.H @@ -46,6 +46,7 @@ static int ppm_temp_fix; static int ppm_predict_gammae; static int ppm_reference_eigenvectors; static int plm_iorder; +static int plm_limiter; static int plm_well_balanced; static int hybrid_riemann; static int riemann_solver; diff --git a/Source/driver/param_includes/castro_queries.H b/Source/driver/param_includes/castro_queries.H index fc1d9c17d1..6e067ff91e 100644 --- a/Source/driver/param_includes/castro_queries.H +++ b/Source/driver/param_includes/castro_queries.H @@ -46,6 +46,7 @@ pp.query("ppm_temp_fix", ppm_temp_fix); pp.query("ppm_predict_gammae", ppm_predict_gammae); pp.query("ppm_reference_eigenvectors", ppm_reference_eigenvectors); pp.query("plm_iorder", plm_iorder); +pp.query("plm_limiter", plm_limiter); pp.query("plm_well_balanced", plm_well_balanced); pp.query("hybrid_riemann", hybrid_riemann); pp.query("riemann_solver", riemann_solver); diff --git a/Source/hydro/slope_nd.F90 b/Source/hydro/slope_nd.F90 index 829a999b14..6c1974bf59 100644 --- a/Source/hydro/slope_nd.F90 +++ b/Source/hydro/slope_nd.F90 @@ -16,7 +16,7 @@ subroutine uslope(lo, hi, idir, & dq, qpd_lo, qpd_hi, & dx, domlo, domhi) - use meth_params_module, only: NQ, plm_iorder, QU, QPRES, QRHO, & + use meth_params_module, only: NQ, plm_iorder, plm_limiter, QU, QPRES, QRHO, & const_grav, plm_well_balanced use amrex_constants_module, only: ZERO, HALF, ONE, TWO, FOUR3RD, FOURTH, SIXTH use prob_params_module, only : Symmetry, physbc_lo, physbc_hi @@ -49,6 +49,7 @@ subroutine uslope(lo, hi, idir, & if (plm_iorder == 1) then + ! first order -- piecewise constant slopes do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -59,6 +60,8 @@ subroutine uslope(lo, hi, idir, & else + ! second-order -- piecewise linear slopes + if (idir == 1) then ! Compute slopes in first coordinate direction do k = lo(3), hi(3) @@ -96,7 +99,7 @@ subroutine uslope(lo, hi, idir, & dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - else if (plm_iorder == 2) then + else if (plm_limiter == 1) then ! the 2nd order MC limiter qm1 = q(i-1,j,k,n) @@ -226,7 +229,7 @@ subroutine uslope(lo, hi, idir, & dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - else if (plm_iorder == 2) then + else if (plm_limiter == 1) then ! the 2nd order MC limiter qm1 = q(i,j-1,k,n) @@ -335,7 +338,7 @@ subroutine uslope(lo, hi, idir, & dq(i,j,k,n) = flatn(i,j,k)*dsgn*min(dlim, abs(dcen)) - else if (plm_iorder == 2) then + else if (plm_limiter == 1) then ! the 2nd order MC limiter qm1 = q(i,j,k-1,n) From 455510f18db2bbdcdc12573325109892a98f14be Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 18 Sep 2019 08:17:49 -0400 Subject: [PATCH 60/72] update notes on plm_iorder --- CHANGES | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 866aed4c3c..15a1e24709 100644 --- a/CHANGES +++ b/CHANGES @@ -14,8 +14,11 @@ changes since the last release -- The weighting of the time-node fluxes stored in the flux registers for SDC has been fixed (#658) - -- The meaning of the castro.plm_iorder flag has changed slightly. - 1 = piecewise constant, 2 = 2nd order MC limiter, and >= 3 for + -- As before, we can choose the reconstruction with PLM using the + castro.plm_iorder flag: 1 = piecewise constant, 2 = piecewise + linear slopes. Now we added a way to specify the limiter used + with the linear slopes. castro.plm_limiter = 1 will use the 2nd + order MC limiter and castro.plm_limiter = 2 will use the default 4th order MC limiter (previously there was no way to select the 2nd order limiter). From ff67ee43a9309bb7fd61adcf2f7b84a6884f0978 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 18 Sep 2019 10:49:14 -0400 Subject: [PATCH 61/72] use constrained_layout --- Exec/science/Detonation/sdc_tests/make_plots.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/make_plots.py b/Exec/science/Detonation/sdc_tests/make_plots.py index 03084071d4..2ce0ae3783 100755 --- a/Exec/science/Detonation/sdc_tests/make_plots.py +++ b/Exec/science/Detonation/sdc_tests/make_plots.py @@ -159,14 +159,13 @@ def get_data(self, n=-1): if len(dset) == 0: continue - fig = plt.figure(1) - fig.clear() + fig, axes = plt.subplots(ncols=1, nrows=3, constrained_layout=True, squeeze=True) fig.set_size_inches(7.0, 9.0) - ax_T = fig.add_subplot(311) - ax_e = fig.add_subplot(312) - ax_he = fig.add_subplot(313) + ax_T = axes[0] + ax_e = axes[1] + ax_he = axes[2] for q in dset: pf = q.get_data(idx) @@ -192,5 +191,4 @@ def get_data(self, n=-1): fig.suptitle("integrator = {}; number of zones = {}".format(intg, nz)) - fig.tight_layout() fig.savefig("det_cfl_compare_{}_nz{}.png".format(intg, nz)) From e2fd1a0f6efe58d84f1a52980dec13b3e7de2da5 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 18 Sep 2019 15:40:29 -0400 Subject: [PATCH 62/72] add SDC-2 --- .../Detonation/sdc_tests/setup_runs.sh | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/Exec/science/Detonation/sdc_tests/setup_runs.sh b/Exec/science/Detonation/sdc_tests/setup_runs.sh index 69ff10e3b2..75487f4275 100755 --- a/Exec/science/Detonation/sdc_tests/setup_runs.sh +++ b/Exec/science/Detonation/sdc_tests/setup_runs.sh @@ -59,6 +59,46 @@ do done +#============================================================================= +# Lobatto SDC-2 +#============================================================================= + +RUNPARAMS=" +castro.time_integration_method=2 +castro.sdc_order=2 +castro.sdc_quadrature=0 +castro.limit_fourth_order=1 +castro.use_reconstructed_gamma1=1 +castro.sdc_solve_for_rhoe=1 +castro.sdc_solver_tol_dens=1.e-8 +castro.sdc_solver_tol_spec=1.e-8 +castro.sdc_solver_tol_ener=1.e-8 +castro.sdc_solver=2" + +for c in ${CFL} +do + + for nz in ${NZONES} + do + rdir=det_z${nz}_c${c}_lobatto_sdc4 + if [ ! -d ${rdir} ]; then + mkdir ${rdir} + fi + + cd ${rdir} + for nf in ${NEEDED_FILES} + do + if [ ! -f ${nf} ]; then + cp ../${nf} . + fi + done + + nohup ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & + cd .. + done +done + + #============================================================================= # Radau SDC-4 #============================================================================= @@ -99,6 +139,46 @@ do done +#============================================================================= +# Radau SDC-2 +#============================================================================= + +RUNPARAMS=" +castro.time_integration_method=2 +castro.sdc_order=2 +castro.sdc_quadrature=1 +castro.limit_fourth_order=1 +castro.use_reconstructed_gamma1=1 +castro.sdc_solve_for_rhoe=1 +castro.sdc_solver_tol_dens=1.e-8 +castro.sdc_solver_tol_spec=1.e-8 +castro.sdc_solver_tol_ener=1.e-8 +castro.sdc_solver=2" + +for c in ${CFL} +do + + for nz in ${NZONES} + do + rdir=det_z${nz}_c${c}_radau_sdc4 + if [ ! -d ${rdir} ]; then + mkdir ${rdir} + fi + + cd ${rdir} + for nf in ${NEEDED_FILES} + do + if [ ! -f ${nf} ]; then + cp ../${nf} . + fi + done + + nohup ${CASTRO_EXEC} inputs.1d.sdc amr.plot_file=${rdir}_plt ${GLOBAL_RUNPARAMS} ${RUNPARAMS} castro.cfl=${c} amr.n_cell=${nz} >& out & + cd .. + done +done + + #============================================================================= # Strang CTU From 4c2610a0f73a75edb047bd5c43c8eff4241a4b80 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Wed, 18 Sep 2019 15:42:18 -0400 Subject: [PATCH 63/72] some tweaks to the inputs --- Exec/science/Detonation/sdc_tests/inputs.1d.sdc | 4 ++-- Exec/science/Detonation/sdc_tests/probin.sdc | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/inputs.1d.sdc b/Exec/science/Detonation/sdc_tests/inputs.1d.sdc index cef4192c7e..69f2ab321e 100644 --- a/Exec/science/Detonation/sdc_tests/inputs.1d.sdc +++ b/Exec/science/Detonation/sdc_tests/inputs.1d.sdc @@ -1,6 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- -max_step = 40000 -stop_time = 0.2 +max_step = 400000 +stop_time = 3.e-6 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 0 0 0 diff --git a/Exec/science/Detonation/sdc_tests/probin.sdc b/Exec/science/Detonation/sdc_tests/probin.sdc index 2a3dad2295..af3c14fbdb 100644 --- a/Exec/science/Detonation/sdc_tests/probin.sdc +++ b/Exec/science/Detonation/sdc_tests/probin.sdc @@ -36,4 +36,6 @@ ! rtol_enuc = 1.e-8 call_eos_in_rhs = T do_constant_volume_burn = T + + small_x = 1.e-10 / From 3a467242d04e65c9728c59e6a4cab61523f8d6b1 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 19 Sep 2019 08:32:50 -0400 Subject: [PATCH 64/72] fix dir names --- Exec/science/Detonation/sdc_tests/setup_runs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/setup_runs.sh b/Exec/science/Detonation/sdc_tests/setup_runs.sh index 75487f4275..a101579f95 100755 --- a/Exec/science/Detonation/sdc_tests/setup_runs.sh +++ b/Exec/science/Detonation/sdc_tests/setup_runs.sh @@ -80,7 +80,7 @@ do for nz in ${NZONES} do - rdir=det_z${nz}_c${c}_lobatto_sdc4 + rdir=det_z${nz}_c${c}_lobatto_sdc2 if [ ! -d ${rdir} ]; then mkdir ${rdir} fi @@ -160,7 +160,7 @@ do for nz in ${NZONES} do - rdir=det_z${nz}_c${c}_radau_sdc4 + rdir=det_z${nz}_c${c}_radau_sdc2 if [ ! -d ${rdir} ]; then mkdir ${rdir} fi From 7fd8983888fd71d13eb92b7f0482a60f8a174722 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 19 Sep 2019 13:20:51 -0400 Subject: [PATCH 65/72] add a plot that groups integrators --- .../Detonation/sdc_tests/make_plots.py | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/make_plots.py b/Exec/science/Detonation/sdc_tests/make_plots.py index f21f14389c..7b5026921f 100755 --- a/Exec/science/Detonation/sdc_tests/make_plots.py +++ b/Exec/science/Detonation/sdc_tests/make_plots.py @@ -153,9 +153,15 @@ def get_data(self, n=-1): runs.sort() print(runs) - # make a plot of the profiles for different CFLs and different resolutions - nzones = set([q.nzones for q in runs]) - integrators = set([q.integrator for q in runs]) + nzones = list(set([q.nzones for q in runs])) + nzones.sort() + integrators = list(set([q.integrator for q in runs])) + integrators.sort() + cfls = list(set([q.cfl for q in runs])) + cfls.sort() + + # make a plot of the profiles for different CFLs with the same + # integrator and resolutions idx = 5 for nz in nzones: @@ -198,3 +204,49 @@ def get_data(self, n=-1): fig.suptitle("{}, number of zones = {}".format(intg, nz)) fig.savefig("det_cfl_compare_{}_nz{}.png".format(intg.replace(" ", "_"), nz)) + plt.close() + + # make a plot of the profiles for different integrators with the same CFLs and resolutions + + idx = 5 + for nz in nzones: + for c in cfls: + dset = [q for q in runs if q.cfl == c and q.nzones == nz] + dset.sort(key=lambda q: q.integrator) + if len(dset) == 0: + continue + + fig, axes = plt.subplots(ncols=1, nrows=3, constrained_layout=True, squeeze=True) + + fig.set_size_inches(7.0, 9.0) + + ax_T = axes[0] + ax_e = axes[1] + ax_he = axes[2] + + for q in dset: + pf = q.get_data(idx) + print("working on {}, time = {}".format(q.name, pf.time)) + + ax_T.plot(pf.x, pf.T, label="{}".format(q.integrator)) + ax_e.plot(pf.x, pf.enuc) + ax_he.plot(pf.x, pf.rho_he4) + + ax_T.legend(frameon=False) + + ax_T.set_ylabel(r"$T$ (K)") + ax_e.set_ylabel(r"$H_\mathrm{nuc}$ (erg/g/s)") + ax_he.set_ylabel(r"$\rho X({}^4\mathrm{He})$ (g/cm${}^3$)") + ax_he.set_xlabel("x (cm)") + + ax_T.set_xlim(5000, 15000) + ax_e.set_xlim(5000, 15000) + ax_he.set_xlim(5000, 15000) + + ax_e.set_yscale("log") + ax_he.set_yscale("log") + + fig.suptitle("number of zones = {}, CFL = {}".format(nz, c)) + + fig.savefig("det_integrator_compare_cfl{}_nz{}.png".format(c, nz)) + plt.close() From 2d6c837d8ecb160baaa1399d6cdf31fbf0691289 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 19 Sep 2019 20:54:06 -0400 Subject: [PATCH 66/72] some robustness to the plot script --- Exec/science/Detonation/sdc_tests/make_plots.py | 12 +++++++++--- Source/hydro/Castro_fourth_order.F90 | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Exec/science/Detonation/sdc_tests/make_plots.py b/Exec/science/Detonation/sdc_tests/make_plots.py index 7b5026921f..cb841ac204 100755 --- a/Exec/science/Detonation/sdc_tests/make_plots.py +++ b/Exec/science/Detonation/sdc_tests/make_plots.py @@ -133,9 +133,10 @@ def get_velocity(self): def get_data(self, n=-1): """get the temperature and energy generation rate from the nth plotfile (starting to count at 0).""" - - return Profile(os.path.join(self.name, self.files[n])) - + try: + return Profile(os.path.join(self.name, self.files[n])) + except IndexError: + return None if __name__ == "__main__": @@ -181,6 +182,9 @@ def get_data(self, n=-1): for q in dset: pf = q.get_data(idx) + if not pf: + continue + print("working on {}, time = {}".format(q.name, pf.time)) ax_T.plot(pf.x, pf.T, label="CFL = {}".format(q.cfl)) @@ -226,6 +230,8 @@ def get_data(self, n=-1): for q in dset: pf = q.get_data(idx) + if not pf: + continue print("working on {}, time = {}".format(q.name, pf.time)) ax_T.plot(pf.x, pf.T, label="{}".format(q.integrator)) diff --git a/Source/hydro/Castro_fourth_order.F90 b/Source/hydro/Castro_fourth_order.F90 index 7cdb051e1c..40b56bc4ba 100644 --- a/Source/hydro/Castro_fourth_order.F90 +++ b/Source/hydro/Castro_fourth_order.F90 @@ -273,6 +273,8 @@ subroutine ca_fourth_single_stage(lo, hi, time, domlo, domhi, & flatn = ONE end if + flatn = ZERO + ! do the reconstruction here -- get the interface states ! x-interfaces From 98203fcadcda8e755961645797548bb7fafc357e Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 20 Sep 2019 19:28:29 -0400 Subject: [PATCH 67/72] undo commit --- Source/hydro/Castro_fourth_order.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/hydro/Castro_fourth_order.F90 b/Source/hydro/Castro_fourth_order.F90 index 40b56bc4ba..7cdb051e1c 100644 --- a/Source/hydro/Castro_fourth_order.F90 +++ b/Source/hydro/Castro_fourth_order.F90 @@ -273,8 +273,6 @@ subroutine ca_fourth_single_stage(lo, hi, time, domlo, domhi, & flatn = ONE end if - flatn = ZERO - ! do the reconstruction here -- get the interface states ! x-interfaces From c123f71ecdd358f9f0f2f747f1b3aa1bce2c7cde Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 23 Sep 2019 09:44:17 -0400 Subject: [PATCH 68/72] remove MethodOfLines --- Source/hydro/Castro_mol_hydro.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index bef51eb067..65ba80b856 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -489,9 +489,8 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // For SDC, we store node 0 the only time we enter here (the // first iteration) and we store the other nodes only on the // last iteration. - if (time_integration_method == MethodOfLines || - (time_integration_method == SpectralDeferredCorrections && - (current_sdc_node == 0 || sdc_iteration == sdc_order+sdc_extra-1))) { + if (time_integration_method == SpectralDeferredCorrections && + (current_sdc_node == 0 || sdc_iteration == sdc_order+sdc_extra-1)) { for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { From f13dcd213de5424495d6deb22e35540dfae93767 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Mon, 23 Sep 2019 09:57:04 -0400 Subject: [PATCH 69/72] move if test out of loop --- Source/hydro/Castro_mol_nd.F90 | 50 +++++++++++++--------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/Source/hydro/Castro_mol_nd.F90 b/Source/hydro/Castro_mol_nd.F90 index 04d532236e..a7c2ace444 100644 --- a/Source/hydro/Castro_mol_nd.F90 +++ b/Source/hydro/Castro_mol_nd.F90 @@ -186,32 +186,28 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & ! special care for reflecting BCs if (idir == 1) then - if (lo(1) == domlo(1)) then + if (lo(1) == domlo(1) .and. physbc_lo(1) == Symmetry) then + ! reset the left state at domlo(1) if needed -- it is outside the domain do k = lo(3)-dg(3), hi(3)+dg(3) do j = lo(2)-dg(2), hi(2)+dg(2) - ! reset the left state at domlo(1) if needed -- it is outside the domain - if (physbc_lo(1) == Symmetry) then - qm(domlo(1),j,k,:,1) = qp(domlo(1),j,k,:,1) - qm(domlo(1),j,k,QU,1) = -qp(domlo(1),j,k,QU,1) - end if + qm(domlo(1),j,k,:,1) = qp(domlo(1),j,k,:,1) + qm(domlo(1),j,k,QU,1) = -qp(domlo(1),j,k,QU,1) end do end do end if - if (hi(1)+1 == domhi(1)+1) then + if (hi(1)+1 == domhi(1)+1 .and. physbc_hi(1) == Symmetry) then ! reset the right state at domhi(1)+1 if needed -- it is outside the domain do k = lo(3)-dg(3), hi(3)+dg(3) do j = lo(2)-dg(2), hi(2)+dg(2) - if (physbc_hi(1) == Symmetry) then - qp(domhi(1)+1,j,k,:,1) = qm(domhi(1)+1,j,k,:,1) - qp(domhi(1)+1,j,k,QU,1) = -qm(domhi(1)+1,j,k,QU,1) - end if + qp(domhi(1)+1,j,k,:,1) = qm(domhi(1)+1,j,k,:,1) + qp(domhi(1)+1,j,k,QU,1) = -qm(domhi(1)+1,j,k,QU,1) end do end do @@ -221,32 +217,28 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & #if AMREX_SPACEDIM >= 2 else if (idir == 2) then - if (lo(2) == domlo(2)) then + if (lo(2) == domlo(2) .and. physbc_lo(2) == Symmetry) then ! reset the left state at domlo(2) if needed -- it is outside the domain do k = lo(3)-dg(3), hi(3)+dg(3) do i = lo(1)-1, hi(1)+1 - if (physbc_lo(2) == Symmetry) then - qm(i,domlo(2),k,:,2) = qp(i,domlo(2),k,:,2) - qm(i,domlo(2),k,QV,2) = -qp(i,domlo(2),k,QV,2) - end if + qm(i,domlo(2),k,:,2) = qp(i,domlo(2),k,:,2) + qm(i,domlo(2),k,QV,2) = -qp(i,domlo(2),k,QV,2) end do end do end if - if (hi(2)+1 == domhi(2)+1) then + if (hi(2)+1 == domhi(2)+1 .and. physbc_hi(2) == Symmetry) then ! reset the right state at domhi(2)+1 if needed -- it is outside the domain do k = lo(3)-dg(3), hi(3)+dg(3) do i = lo(1)-1, hi(1)+1 - if (physbc_hi(2) == Symmetry) then - qp(i,domhi(2)+1,k,:,2) = qm(i,domhi(2)+1,k,:,2) - qp(i,domhi(2)+1,k,QV,2) = -qm(i,domhi(2)+1,k,QV,2) - end if + qp(i,domhi(2)+1,k,:,2) = qm(i,domhi(2)+1,k,:,2) + qp(i,domhi(2)+1,k,QV,2) = -qm(i,domhi(2)+1,k,QV,2) end do end do @@ -256,32 +248,28 @@ subroutine ca_mol_plm_reconstruct(lo, hi, & #if AMREX_SPACEDIM == 3 else - if (lo(3) == domlo(3)) then + if (lo(3) == domlo(3) .and. physbc_lo(3) == Symmetry) then ! reset the left state at domlo(3) if needed -- it is outside the domain do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - if (physbc_lo(3) == Symmetry) then - qm(i,j,domlo(3),:,3) = qp(i,j,domlo(3),:,3) - qm(i,j,domlo(3),QW,3) = -qp(i,j,domlo(3),QW,3) - end if + qm(i,j,domlo(3),:,3) = qp(i,j,domlo(3),:,3) + qm(i,j,domlo(3),QW,3) = -qp(i,j,domlo(3),QW,3) end do end do end if - if (hi(3)+1 == domhi(3)+1) then + if (hi(3)+1 == domhi(3)+1 .and. physbc_hi(3) == Symmetry) then ! reset the right state at domhi(3)+1 if needed -- it is outside the domain do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - if (physbc_hi(3) == Symmetry) then - qp(i,j,domhi(3)+1,:,3) = qm(i,j,domhi(3)+1,:,3) - qp(i,j,domhi(3)+1,QW,3) = -qm(i,j,domhi(3)+1,QW,3) - end if + qp(i,j,domhi(3)+1,:,3) = qm(i,j,domhi(3)+1,:,3) + qp(i,j,domhi(3)+1,QW,3) = -qm(i,j,domhi(3)+1,QW,3) end do end do From 571f1f0c07198078a7ec2fb44a54d9e5c992f90d Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 24 Sep 2019 12:12:45 -0400 Subject: [PATCH 70/72] fill U_old --- Source/driver/sdc_util.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index 9998b9ea3e..f8c2130770 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -75,6 +75,8 @@ subroutine sdc_solve(dt_m, U_old, U_new, C, sdc_iteration) ! for debugging real(rt) :: U_orig(NVAR) + U_orig(:) = U_old(:) + if (sdc_solver == NEWTON_SOLVE) then ! we are going to assume we already have a good guess for the ! solving in U_new and just pass the solve onto the main Newton From a919557009ea5ff24e76a7523b41a6d9b2e2003b Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 24 Sep 2019 14:35:47 -0400 Subject: [PATCH 71/72] change precision of weights --- Source/driver/Castro_setup.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/driver/Castro_setup.cpp b/Source/driver/Castro_setup.cpp index 0787a7e503..5850405680 100644 --- a/Source/driver/Castro_setup.cpp +++ b/Source/driver/Castro_setup.cpp @@ -1134,10 +1134,10 @@ Castro::variableSetUp () SDC_NODES = 4; dt_sdc.resize(SDC_NODES); - dt_sdc = {0.0, (4.0 - std::sqrt(6))/10.0, (4.0 + std::sqrt(6))/10.0, 1.0}; + dt_sdc = {0.0, (4.0 - std::sqrt(6.0))/10.0, (4.0 + std::sqrt(6.0))/10.0, 1.0}; node_weights.resize(SDC_NODES); - node_weights = {0.0, (16.0 - std::sqrt(6))/36.0, (16.0 + std::sqrt(6))/36.0, 1.0/9.0}; + node_weights = {0.0, (16.0 - std::sqrt(6.0))/36.0, (16.0 + std::sqrt(6.0))/36.0, 1.0/9.0}; } else { amrex::Error("invalid value of sdc_order"); From 022d2e1356ffd54a49b0f1ab73616f838194f96e Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Tue, 24 Sep 2019 14:43:17 -0400 Subject: [PATCH 72/72] fix merge issue --- Source/driver/sdc_util.F90 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/driver/sdc_util.F90 b/Source/driver/sdc_util.F90 index f8c2130770..21489e4943 100644 --- a/Source/driver/sdc_util.F90 +++ b/Source/driver/sdc_util.F90 @@ -1508,11 +1508,6 @@ subroutine ca_sdc_update_o2(lo, hi, dt_m, & U_old(:) = k_m(i,j,k,:) C_zone(:) = C(i,j,k,:) - ! only burn if we are within the temperature and density - ! limits for burning - if (.not. okay_to_burn(U_old)) then - R_full(:) = ZERO - ! only burn if we are within the temperature and density ! limits for burning if (.not. okay_to_burn(U_old)) then