Skip to content

Commit

Permalink
merge the transverse routines (#768)
Browse files Browse the repository at this point in the history 
This consolidates the various transverse routines, using mini-Castro as a guide.

* 2-d axisymmetric pure hydro will have roundoff diffs because we now include the area factors in the z-direction (previously it was only in the r direction)

* we are slower by about 10% -- this is likely because the compiler cannot do cache optimization. We will address this with templating when we convert to C++

* for radiation, in 2-d we now use the general geometry form for the radiation pressure in the transverse flux
  • Loading branch information
@zingale
zingale authored Feb 21, 2020
1 parent 972b99d commit 45994be
Show file tree
Hide file tree
Showing 5 changed files with 928 additions and 4,403 deletions.
255 changes: 134 additions & 121 deletions Source/hydro/Castro_ctu_hydro.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,18 +517,21 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// ftmp2 = fy
// rftmp2 = rfy
#pragma gpu box(xbx)
transy_on_xstates(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp2),
trans_single(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
2, 1,
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp2),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp2),
BL_TO_FORTRAN_ANYD(rftmp2),
#endif
BL_TO_FORTRAN_ANYD(qe[1]),
hdtdy);
BL_TO_FORTRAN_ANYD(qe[1]),
BL_TO_FORTRAN_ANYD(area[1][mfi]),
BL_TO_FORTRAN_ANYD(volume[mfi]),
hdt, hdtdy);

#pragma gpu box(xbx)
reset_edge_state_thermo(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
Expand Down Expand Up @@ -563,20 +566,21 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// qgdnvtmp1 = qgdnvx

#pragma gpu box(ybx)
transx_on_ystates(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
1, 2,
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(area[0][mfi]),
BL_TO_FORTRAN_ANYD(volume[mfi]),
hdt, hdtdx);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(area[0][mfi]),
BL_TO_FORTRAN_ANYD(volume[mfi]),
hdt, hdtdx);

#pragma gpu box(ybx)
reset_edge_state_thermo(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
Expand Down Expand Up @@ -656,18 +660,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// rftmp1 = rfx
// qgdnvtmp1 = qgdnvx
#pragma gpu box(tyxbx)
transx_on_ystates(AMREX_INT_ANYD(tyxbx.loVect()), AMREX_INT_ANYD(tyxbx.hiVect()),
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(qmyx),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qpyx),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(tyxbx.loVect()), AMREX_INT_ANYD(tyxbx.hiVect()),
1, 2,
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(qmyx),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qpyx),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdx);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdx);

#pragma gpu box(tyxbx)
reset_edge_state_thermo(AMREX_INT_ANYD(tyxbx.loVect()), AMREX_INT_ANYD(tyxbx.hiVect()),
Expand All @@ -689,18 +694,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
fab_size += qpzx.nBytes();

#pragma gpu box(tzxbx)
transx_on_zstates(AMREX_INT_ANYD(tzxbx.loVect()), AMREX_INT_ANYD(tzxbx.hiVect()),
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(qmzx),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qpzx),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(tzxbx.loVect()), AMREX_INT_ANYD(tzxbx.hiVect()),
1, 3,
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(qmzx),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qpzx),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdx);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdx);

#pragma gpu box(tzxbx)
reset_edge_state_thermo(AMREX_INT_ANYD(tzxbx.loVect()), AMREX_INT_ANYD(tzxbx.hiVect()),
Expand Down Expand Up @@ -747,18 +753,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// rftmp1 = rfy
// qgdnvtmp1 = qgdnvy
#pragma gpu box(txybx)
transy_on_xstates(AMREX_INT_ANYD(txybx.loVect()), AMREX_INT_ANYD(txybx.hiVect()),
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(qmxy),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qpxy),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(txybx.loVect()), AMREX_INT_ANYD(txybx.hiVect()),
2, 1,
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(qmxy),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qpxy),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
cdtdy);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdy);

#pragma gpu box(txybx)
reset_edge_state_thermo(AMREX_INT_ANYD(txybx.loVect()), AMREX_INT_ANYD(txybx.hiVect()),
Expand All @@ -783,18 +790,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// rftmp1 = rfy
// qgdnvtmp1 = qgdnvy
#pragma gpu box(tzybx)
transy_on_zstates(AMREX_INT_ANYD(tzybx.loVect()), AMREX_INT_ANYD(tzybx.hiVect()),
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(qmzy),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qpzy),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(tzybx.loVect()), AMREX_INT_ANYD(tzybx.hiVect()),
2, 3,
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(qmzy),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qpzy),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
cdtdy);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdy);

#pragma gpu box(tzybx)
reset_edge_state_thermo(AMREX_INT_ANYD(tzybx.loVect()), AMREX_INT_ANYD(tzybx.hiVect()),
Expand Down Expand Up @@ -841,18 +849,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// rftmp1 = rfz
// qgdnvtmp1 = qgdnvz
#pragma gpu box(txzbx)
transz_on_xstates(AMREX_INT_ANYD(txzbx.loVect()), AMREX_INT_ANYD(txzbx.hiVect()),
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(qmxz),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qpxz),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(txzbx.loVect()), AMREX_INT_ANYD(txzbx.hiVect()),
3, 1,
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(qmxz),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qpxz),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
cdtdz);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdz);

#pragma gpu box(txzbx)
reset_edge_state_thermo(AMREX_INT_ANYD(txzbx.loVect()), AMREX_INT_ANYD(txzbx.hiVect()),
Expand All @@ -877,18 +886,19 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// rftmp1 = rfz
// qgdnvtmp1 = qgdnvz
#pragma gpu box(tyzbx)
transz_on_ystates(AMREX_INT_ANYD(tyzbx.loVect()), AMREX_INT_ANYD(tyzbx.hiVect()),
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(qmyz),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qpyz),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_single(AMREX_INT_ANYD(tyzbx.loVect()), AMREX_INT_ANYD(tyzbx.hiVect()),
3, 2,
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(qmyz),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qpyz),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
cdtdz);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, cdtdz);

#pragma gpu box(tyzbx)
reset_edge_state_thermo(AMREX_INT_ANYD(tyzbx.loVect()), AMREX_INT_ANYD(tyzbx.hiVect()),
Expand Down Expand Up @@ -952,23 +962,24 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// [lo(1), lo(2), lo(3)], [hi(1)+1, hi(2), hi(3)]

#pragma gpu box(xbx)
transyz(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_final(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
1, 2, 3,
BL_TO_FORTRAN_ANYD(qxm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qxp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(ftmp2),
BL_TO_FORTRAN_ANYD(ftmp2),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp2),
BL_TO_FORTRAN_ANYD(rftmp2),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
hdt, hdtdy, hdtdz);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
hdt, hdtdx, hdtdy, hdtdz);

#pragma gpu box(xbx)
reset_edge_state_thermo(AMREX_INT_ANYD(xbx.loVect()), AMREX_INT_ANYD(xbx.hiVect()),
Expand Down Expand Up @@ -1045,23 +1056,24 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// [lo(1), lo(2), lo(3)], [hi(1), hi(2)+1, hi(3)]

#pragma gpu box(ybx)
transxz(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp2),
trans_final(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
2, 1, 3,
BL_TO_FORTRAN_ANYD(qym),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qyp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp2),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp2),
BL_TO_FORTRAN_ANYD(rftmp2),
#endif
BL_TO_FORTRAN_ANYD(ftmp1),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, hdtdx, hdtdz);
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
hdt, hdtdx, hdtdy, hdtdz);

#pragma gpu box(ybx)
reset_edge_state_thermo(AMREX_INT_ANYD(ybx.loVect()), AMREX_INT_ANYD(ybx.hiVect()),
Expand Down Expand Up @@ -1140,23 +1152,24 @@ Castro::construct_ctu_hydro_source(Real time, Real dt)
// [lo(1), lo(2), lo(3)], [hi(1), hi(2), hi(3)+1]

#pragma gpu box(zbx)
transxy(AMREX_INT_ANYD(zbx.loVect()), AMREX_INT_ANYD(zbx.hiVect()),
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
trans_final(AMREX_INT_ANYD(zbx.loVect()), AMREX_INT_ANYD(zbx.hiVect()),
3, 1, 2,
BL_TO_FORTRAN_ANYD(qzm),
BL_TO_FORTRAN_ANYD(ql),
BL_TO_FORTRAN_ANYD(qzp),
BL_TO_FORTRAN_ANYD(qr),
BL_TO_FORTRAN_ANYD(qaux[mfi]),
BL_TO_FORTRAN_ANYD(ftmp1),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp1),
BL_TO_FORTRAN_ANYD(rftmp1),
#endif
BL_TO_FORTRAN_ANYD(ftmp2),
BL_TO_FORTRAN_ANYD(ftmp2),
#ifdef RADIATION
BL_TO_FORTRAN_ANYD(rftmp2),
BL_TO_FORTRAN_ANYD(rftmp2),
#endif
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
hdt, hdtdx, hdtdy);
BL_TO_FORTRAN_ANYD(qgdnvtmp1),
BL_TO_FORTRAN_ANYD(qgdnvtmp2),
hdt, hdtdx, hdtdy, hdtdz);

#pragma gpu box(zbx)
reset_edge_state_thermo(AMREX_INT_ANYD(zbx.loVect()), AMREX_INT_ANYD(zbx.hiVect()),
Expand Down
Loading

0 comments on commit 45994be

Please sign in to comment.