main3d.F

Go to the documentation of this file.

#include "cppdefs.h"
#if defined NONLINEAR && defined SOLVE3D
      SUBROUTINE main3d (RunInterval)
!
!git $Id$
!=======================================================================
!  Copyright (c) 2002-2025 The ROMS Group                              !
!    Licensed under a MIT/X style license                              !
!    See License_ROMS.md                            Hernan G. Arango   !
!========================================== Alexander F. Shchepetkin ===
!                                                                      !
!  This routine is the main driver for ROMS nonlinear model (NLM)      !
!  when configurated as a full 3D baroclinic ocean model. It advances  !
!  forward the NLM for all nested grids, if any, for the specified     !
!  time interval (seconds), RunInterval.                               !
!                                                                      !
# if defined STEP2D_FB_LF_AM3
!  Numerical 2D time-stepping kernel: FB AB3-AM4                       !
# elif defined STEP2D_FB_LF_AM3
!  Numerical 2D time-stepping kernel: FB LF-AM3                        !
# else
!  Numerical 2D time-stepping kernel: LF-AM3 (Legacy scheme)           !
# endif
!                                                                      !
!=======================================================================
!
      USE mod_param
      USE mod_parallel
# if defined MODEL_COUPLING && defined MCT_LIB
      USE mod_coupler
# endif
      USE mod_iounits
# ifdef NESTING
      USE mod_nesting
# endif
      USE mod_scalars
      USE mod_stepping
!
# ifdef ANA_VMIX
      USE analytical_mod,       ONLY : ana_vmix
# endif
# ifdef BIOLOGY
      USE biology_mod,          ONLY : biology
# endif
# ifdef BBL_MODEL
      USE bbl_mod,              ONLY : bblm
# endif
# ifdef BULK_FLUXES
      USE bulk_flux_mod,        ONLY : bulk_flux
# endif
# ifdef BVF_MIXING
      USE bvf_mix_mod,          ONLY : bvf_mix
# endif
      USE dateclock_mod,        ONLY : time_string
      USE diag_mod,             ONLY : diag
# ifdef TLM_CHECK
      USE dotproduct_mod,       ONLY : nl_dotproduct
# endif
# ifdef TIDE_GENERATING_FORCES
      USE equilibrium_tide_mod, ONLY : equilibrium_tide
# endif
# if defined NLM_OUTER                || \
     defined rbl4dvar                 || \
     defined rbl4dvar_ana_sensitivity || \
     defined rbl4dvar_fct_sensitivity || \
     defined sp4dvar
      USE forcing_mod,          ONLY : forcing
# endif
# if defined ADJUST_STFLUX || defined ADJUST_WSTRESS
      USE frc_adjust_mod,       ONLY : frc_adjust, load_frc
# endif
# ifdef GLS_MIXING
      USE gls_corstep_mod,      ONLY : gls_corstep
      USE gls_prestep_mod,      ONLY : gls_prestep
# endif
# if defined NLM_OUTER             || defined RBL4DVAR                 || \
  defined rbl4dvar_ana_sensitivity || defined rbl4dvar_fct_sensitivity
      USE frc_iau_mod,          ONLY : frc_iau
# endif
# if defined DIFF_3DCOEF || defined VISC_3DCOEF
      USE hmixing_mod,          ONLY : hmixing
# endif
# if defined ICE_MODEL && defined ALBEDO && defined SHORTWAVE
      USE ice_albedo_mod,       ONLY : ice_albedo
# endif
# ifdef LMD_MIXING
      USE lmd_vmix_mod,         ONLY : lmd_vmix
# endif
# if defined ATM_COUPLING && defined MCT_LIB
      USE mct_coupler_mod,      ONLY : ocn2atm_coupling
# endif
# if defined WAV_COUPLING && defined MCT_LIB
      USE mct_coupler_mod,      ONLY : ocn2wav_coupling
# endif
# ifdef MY25_MIXING
      USE my25_corstep_mod,     ONLY : my25_corstep
      USE my25_prestep_mod,     ONLY : my25_prestep
# endif
# ifdef NESTING
      USE nesting_mod,          ONLY : nesting
#  ifndef ONE_WAY
      USE nesting_mod,          ONLY : do_twoway
#  endif
# endif
# if defined ADJUST_BOUNDARY
      USE obc_adjust_mod,       ONLY : obc_adjust, load_obc
# endif
      USE omega_mod,            ONLY : omega
      USE post_initial_mod,     ONLY : post_initial
# ifndef TS_FIXED
      USE rho_eos_mod,          ONLY : rho_eos
# endif
      USE rhs3d_mod,            ONLY : rhs3d
# ifdef ICE_MODEL
      USE seaice_mod,           ONLY : seaice
# endif
# ifdef SEDIMENT
      USE sediment_mod,         ONLY : sediment
# endif
# ifdef AVERAGES
      USE set_avg_mod,          ONLY : set_avg
# endif
      USE set_depth_mod,        ONLY : set_depth
      USE set_massflux_mod,     ONLY : set_massflux
# if defined SSH_TIDES || defined UV_TIDES
      USE set_tides_mod,        ONLY : set_tides
# endif
      USE set_vbc_mod,          ONLY : set_vbc
# if !(defined STEP2D_FB_AB3_AM4 || defined STEP2D_FB_LF_AM3)
      USE set_zeta_mod,         ONLY : set_zeta
# endif
      USE step2d_mod,           ONLY : step2d
# ifndef TS_FIXED
      USE step3d_t_mod,         ONLY : step3d_t
# endif
      USE step3d_uv_mod,        ONLY : step3d_uv
# ifdef FLOATS
      USE step_floats_mod,      ONLY : step_floats
# endif
      USE strings_mod,          ONLY : founderror
# ifdef WEC_VF
#  if defined WDISS_THORGUZA || defined WDISS_CHURTHOR
      USE wec_dissip_mod,       ONLY : wec_dissip
#  endif
#  ifdef WEC_ROLLER
      USE wec_roller_mod,       ONLY : wec_roller
#  endif
      USE wec_stokes_mod,       ONLY : wec_stokes
      USE wec_vf_mod,           ONLY : wec_vf
#  ifdef WAVE_MIXING
      USE wec_wave_mix_mod,     ONLY : wec_wave_mix
#  endif
# endif
# ifdef WEC
      USE wec_wvelocity_mod,    ONLY : wec_wvelocity
# endif
      USE wvelocity_mod,        ONLY : wvelocity
!
      implicit none
!
!  Imported variable declarations.
!
      real(dp), intent(in) :: RunInterval
!
!  Local variable declarations.
!
      logical :: DoNestLayer, Time_Step
!
      integer :: Nsteps, Rsteps
      integer :: ig, il, istep, ng, nl, tile
      integer :: my_iif, next_indx1
# ifdef FLOATS
      integer :: Lend, Lstr, chunk_size
# endif
!
      character (len=*), parameter :: MyFile =                          &
     &  __FILE__
!
!=======================================================================
!  Time-step nonlinear 3D primitive equations by the specified time.
!=======================================================================
!
!  Time-step the 3D kernel for the specified time interval (seconds),
!  RunInterval.
!
      time_step=.true.
      donestlayer=.true.
!
      kernel_loop : DO WHILE (time_step)
!
!  In nesting applications, the number of nesting layers (NestLayers) is
!  used to facilitate refinement grids and composite/refinament grids
!  combinations. Otherwise, the solution it is looped once for a single
!  grid application (NestLayers = 1).
!
        nl=0
#ifdef NESTING
        twowayinterval(1:ngrids)=0.0_r8
#endif
!
        nest_layer : DO WHILE (donestlayer)
!
!  Determine number of time steps to compute in each nested grid layer
!  based on the specified time interval (seconds), RunInterval. Non
!  nesting applications have NestLayers=1. Notice that RunInterval is
!  set in the calling driver. Its value may span the full period of the
!  simulation, a multi-model coupling interval (RunInterval > ifac*dt),
!  or just a single step (RunInterval=0).
!
          CALL ntimesteps (inlm, runinterval, nl, nsteps, rsteps)
          IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
          IF ((nl.le.0).or.(nl.gt.nestlayers)) EXIT
!
!  Time-step governing equations for Nsteps.
!
          step_loop : DO istep=1,nsteps
!
!  Set time indices and time clock.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              nstp(ng)=1+mod(iic(ng)-ntstart(ng),2)
              nnew(ng)=3-nstp(ng)
              nrhs(ng)=nstp(ng)
# ifdef JEDI
              jic(ng)=jic(ng)+1
              time4jedi(ng)=time4jedi(ng)+dt(ng)
# endif
              tdays(ng)=time(ng)*sec2day
              IF (step_counter(ng).eq.rsteps) time_step=.false.
            END DO
!
!-----------------------------------------------------------------------
!  Read in required data, if any, from input NetCDF files.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
!$OMP MASTER
              IF (processinputdata(ng)) THEN
                CALL get_data (ng)
              END IF
!$OMP END MASTER
!$OMP BARRIER
              IF (founderror(exit_flag, noerror,                        &
     &                       __line__, myfile)) RETURN
            END DO
!
!-----------------------------------------------------------------------
!  If applicable, process input data: time interpolate between data
!  snapshots.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (processinputdata(ng)) THEN
                DO tile=first_tile(ng),last_tile(ng),+1
                  CALL set_data (ng, tile)
                END DO
              END IF
!$OMP BARRIER
            END DO
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
# if defined NLM_OUTER                || \
     defined rbl4dvar                 || \
     defined rbl4dvar_ana_sensitivity || \
     defined rbl4dvar_fct_sensitivity
!
!-----------------------------------------------------------------------
!  If appropriate, add convolved adjoint solution impulse forcing to
!  the nonlinear model solution. Notice that the forcing is only needed
!  after finishing all the inner loops. The forcing is continuous.
!  That is, it is time interpolated at every time-step from available
!  snapshots (FrequentImpulse=TRUE).
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
#  ifdef RPCG
              IF ((iic(ng).gt.int(timeiau(ng)/dt(ng))).or.              &
     &            timeiau(ng).eq.0.0_dp) THEN
                iauswitch(ng)=.false.
              END IF
              IF (frequentimpulse(ng).or.iauswitch(ng)) THEN
#  endif
#  if defined WEAK_NOINTERP
                IF ((iic(ng).gt.1).and.(iic(ng).ne.ntend(ng)+1).and.    &
     &              (mod(iic(ng)-1,nadj(ng)).eq.0)) THEN
                  IF (master) THEN
                    WRITE (stdout,*) ' FORCING NLM at iic = ',iic(ng)
                  END IF
#  endif
#  ifdef RPCG
                  IF (frequentimpulse(ng).and.iauswitch(ng)) THEN
                    DO tile=first_tile(ng),last_tile(ng),+1
                    CALL frc_iau (ng, tile, 1)
                    END DO
                  END IF
                  DO tile=first_tile(ng),last_tile(ng),+1
                    CALL forcing (ng, tile, kstp(ng), nstp(ng))
                    CALL set_depth (ng, tile, inlm)
                  END DO
!$OMP BARRIER
#  else
                IF (frequentimpulse(ng)) THEN
                  DO tile=first_tile(ng),last_tile(ng),+1
                    CALL forcing (ng, tile, kstp(ng), nstp(ng))
                    CALL set_depth (ng, tile, inlm)
                  END DO
!$OMP BARRIER
                END IF
#  endif
#  if defined WEAK_NOINTERP
                END IF
#  endif
#  ifdef RPCG
              END IF
#  endif
            END DO
# endif
 
# ifndef JEDI
!
!-----------------------------------------------------------------------
!  On the first timestep, it computes the initial depths and level
!  thicknesses from the initial free-surface field. Additionally, it
!  initializes the nonlinear state variables for all time levels and
!  applies lateral boundary conditions.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (iic(ng).eq.ntstart(ng)) THEN
                CALL post_initial (ng, inlm)
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Compute horizontal mass fluxes (Hz*u/n and Hz*v/m), density related
!  quatities and report global diagnostics.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=first_tile(ng),last_tile(ng),+1
                CALL set_massflux (ng, tile, inlm)
# ifndef TS_FIXED
                CALL rho_eos (ng, tile, inlm)
# endif
# ifdef TIDE_GENERATING_FORCES
                CALL equilibrium_tide (ng, tile, inlm)
# endif
                CALL diag (ng, tile)
# ifdef TLM_CHECK
                CALL nl_dotproduct (ng, tile, lnew(ng))
# endif
              END DO
!$OMP BARRIER
            END DO
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
# if defined ATM_COUPLING && defined MCT_LIB
!
!-----------------------------------------------------------------------
!  Couple ocean to atmosphere model every "CoupleSteps(Iatmos)"
!  timesteps: get air/sea fluxes.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF ((iic(ng).ne.ntstart(ng)).and.                         &
     &            mod(iic(ng)-1,couplesteps(iatmos,ng)).eq.0) THEN
                DO tile=last_tile(ng),first_tile(ng),-1
                  CALL ocn2atm_coupling (ng, tile)
                END DO
!$OMP BARRIER
              END IF
            END DO
# endif
 
# if defined WAV_COUPLING && defined MCT_LIB
!
!-----------------------------------------------------------------------
!  Couple to ocean to waves model every "CoupleSteps(Iwaves)"
!  timesteps: get waves/ocean fluxes.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF ((iic(ng).ne.ntstart(ng)).and.                         &
     &            mod(iic(ng)-1,couplesteps(iwaves,ng)).eq.0) THEN
                DO tile=first_tile(ng),last_tile(ng),+1
                  CALL ocn2wav_coupling (ng, tile)
                END DO
!$OMP BARRIER
              END IF
            END DO
# endif
 
# ifdef WEC_VF
!
!-----------------------------------------------------------------------
!  Compute wave dissipation, roller, and stokes terms.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=first_tile(ng),last_tile(ng),+1
#  if defined WDISS_THORGUZA || defined WDISS_CHURTHOR
                CALL wec_dissip (ng, tile)
#  endif
#  ifdef WEC_ROLLER
                CALL wec_roller (ng, tile)
#  endif
                CALL wec_stokes (ng, tile)
              END DO
!$OMP BARRIER
            END DO
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
# endif
!
!-----------------------------------------------------------------------
!  Set fields for vertical boundary conditions. Process tidal forcing,
!  if any.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=first_tile(ng),last_tile(ng),+1
# ifdef BULK_FLUXES
#  if defined ICE_MODEL && defined ALBEDO && defined SHORTWAVE
                CALL ice_albedo (ng, tile, inlm)
#  endif
#  if defined FOUR_DVAR && defined PRIOR_BULK_FLUXES
                IF (nrun.eq.1) CALL bulk_flux (ng, tile)
#  else
                CALL bulk_flux (ng, tile)
#  endif
# endif
# ifdef BBL_MODEL
                CALL bblm (ng, tile)
# endif
                CALL set_vbc (ng, tile)
# if defined SSH_TIDES || defined UV_TIDES
                CALL set_tides (ng, tile)
# endif
              END DO
!$OMP BARRIER
            END DO
 
# ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for bottom stress variables.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, nbstr)
              END IF
            END DO
# endif
 
# if defined ICE_MODEL
!
!-----------------------------------------------------------------------
!  Timestep the ice model.
!-----------------------------------------------------------------------
!
            CALL seaice (inlm)
# endif
 
# ifdef ADJUST_BOUNDARY
!
!-----------------------------------------------------------------------
!  Interpolate open boundary increments and adjust open boundary.
!  Load open boundary into storage arrays. Skip the last output
!  timestep.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (iic(ng).lt.(ntend(ng)+1)) THEN
                DO tile=first_tile(ng),last_tile(ng),+1
                  CALL obc_adjust (ng, tile, lbinp(ng))
                  CALL load_obc (ng, tile, lbout(ng))
                END DO
!$OMP BARRIER
              END IF
            END DO
# endif
 
# if defined ADJUST_STFLUX || defined ADJUST_WSTRESS
!
!-----------------------------------------------------------------------
!  Interpolate surface forcing increments and adjust surface forcing.
!  Load surface forcing into storage arrays. Skip the last output
!  timestep.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (iic(ng).lt.(ntend(ng)+1)) THEN
                DO tile=first_tile(ng),last_tile(ng),+1
                  CALL frc_adjust (ng, tile, lfinp(ng))
                  CALL load_frc (ng, tile, lfout(ng))
                END DO
!$OMP BARRIER
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Compute time-dependent vertical/horizontal mixing coefficients for
!  momentum and tracers. Compute S-coordinate vertical velocity,
!  diagnostically from horizontal mass divergence.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=last_tile(ng),first_tile(ng),-1
# if defined ANA_VMIX
                CALL ana_vmix (ng, tile, inlm)
# elif defined LMD_MIXING
                CALL lmd_vmix (ng, tile)
# elif defined BVF_MIXING
                CALL bvf_mix (ng, tile)
# endif
# if defined DIFF_3DCOEF || defined VISC_3DCOEF
                CALL hmixing (ng, tile)
# endif
                CALL omega (ng, tile, inlm)
                CALL wvelocity (ng, tile, nstp(ng))
# ifdef WEC
                CALL wec_wvelocity (ng, tile, nstp(ng))
# endif
              END DO
!$OMP BARRIER
            END DO
 
# if !(defined STEP2D_FB_AB3_AM4 || defined STEP2D_FB_LF_AM3) || \
       defined diagnostics       || defined averages
!
!-----------------------------------------------------------------------
!  Set free-surface to it time-averaged value.  If applicable,
!  accumulate time-averaged output data which needs a irreversible
!  loop in shared-memory jobs.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=first_tile(ng),last_tile(ng),+1     ! irreversible
#  if !(defined STEP2D_FB_AB3_AM4 || defined STEP2D_FB_LF_AM3)
                CALL set_zeta (ng, tile)
#  endif
#  ifdef DIAGNOSTICS
                CALL set_diags (ng, tile)
#  endif
#  ifdef AVERAGES
                CALL set_avg (ng, tile)
#  endif
              END DO
!$OMP BARRIER
            END DO
# endif
 
# ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for 3D kernel free-surface.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, nzeta)
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  If appropriate, write out fields into output NetCDF files.  Notice
!  that IO data is written in delayed and serial mode.  Exit if last
!  time step.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
!$OMP MASTER
              CALL output (ng)
!$OMP END MASTER
!$OMP BARRIER
              IF ((founderror(exit_flag, noerror, __line__, myfile)).or.&
     &            ((iic(ng).eq.(ntend(ng)+1)).and.(ng.eq.ngrids))) THEN
                RETURN
              END IF
            END DO
 
# ifdef NESTING
!
!-----------------------------------------------------------------------
!  If refinement grid, interpolate (space, time) state variables
!  contact points from donor grid extracted data.
#  ifdef NESTING_DEBUG
!
!  Also, fill BRY_CONTACT(:,:)%Mflux to check for mass conservation
!  between coarse and fine grids.  This is only done for diagnostic
!  purposes. Also, debugging is possible with very verbose output
!  to fort.300 is allowed by activating uppercase(nesting_debug).
#  endif
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (refinedgrid(ng).and.(refinescale(ng).gt.0)) THEN
                CALL nesting (ng, inlm, nputd)
#  ifdef NESTING_DEBUG
                CALL nesting (ng, inlm, nmflx)
#  endif
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Compute right-hand-side terms for 3D equations.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=last_tile(ng),first_tile(ng),-1
                CALL rhs3d (ng, tile)
# ifdef MY25_MIXING
                CALL my25_prestep (ng, tile)
# elif defined GLS_MIXING
                CALL gls_prestep (ng, tile)
# endif
              END DO
!$OMP BARRIER
            END DO
 
# ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for right-hand-side terms
!  (tracers).
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, nrhst)
              END IF
            END DO
# endif
 
# ifdef STEP2D_FB_AB3_AM4
!
!-----------------------------------------------------------------------
!  Solve nonlinear vertically integrated primitive equations for
!  free-surface and momentum components using a generalized Forward-
!  Backward, 3rd-order Adams-Bashforth / 4th-order Adams-Moulton
!  (FB AB3-AM4) time stepping scheme (Shchepetkin and McWilliams,
!  2009).
!-----------------------------------------------------------------------
!
            loop_2d : DO my_iif=1,maxval(nfast)
 
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (my_iif.le.nfast(ng)) THEN
                  iif(ng)=my_iif
                  kstp(ng)=knew(ng)
                  knew(ng)=kstp(ng)+1
                  IF (knew(ng).gt.4) knew(ng)=1
 
                  IF (mod(knew(ng),2).eq.0) THEN            ! zig-zag
                    DO tile=first_tile(ng),last_tile(ng),+1 ! processing
                      CALL step2d (ng, tile)                ! sequence
                    END DO
!$OMP BARRIER
                  ELSE
                    DO tile=last_tile(ng),first_tile(ng),-1
                      CALL step2d (ng, tile)
                    END DO
!$OMP BARRIER
                  END IF
                END IF
              END DO
 
#  ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the state variables
!  associated with the 2D engine CORRECTOR STEP section (KNEW INDEX).
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (any(compositegrid(:,ng))) THEN
                  CALL nesting (ng, inlm, n2dcs)
                END IF
              END DO
#  endif
            END DO loop_2d
 
# else
 
#  ifdef STEP2D_FB_LF_AM3
!
!-----------------------------------------------------------------------
!  Solve the vertically integrated primitive equations for the
!  free-surface and barotropic momentum components using a predictor-
!  corrector LeapFrog / 3rd-order Adams-Moulton with a Forward-Backward
!  feeback (FB LF-AM3) stepping scheme (Shchepetkin and McWilliams,
!  2009).
!-----------------------------------------------------------------------
!
            loop_2d : DO my_iif=1,maxval(nfast)
!
!  Predictor LF substep with FB-feedback.
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (my_iif.le.nfast(ng)) THEN
                  iif(ng)=my_iif
                  kstp(ng)=next_kstp(ng)
                  knew(ng)=3
 
                  DO tile=last_tile(ng),first_tile(ng),-1
                    CALL step2d (ng, tile)
                  END DO
!$OMP BARRIER
                END IF
              END DO
 
#   ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the state variables
!  associated with the 2D engine PREDICTOR STEP section.
#    ifdef NESTING_DEBUG
!  If refinement, check mass flux conservation between coarse and
!  fine grids during debugging.
!  Warning: very verbose output to fort.300 ascii file to check
!           mass flux conservation.
#    endif
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (any(compositegrid(:,ng))) THEN
                  CALL nesting (ng, inlm, n2dps)
                END IF
#    ifdef NESTING_DEBUG
                IF (refinedgrid(ng).and.(refinescale(ng).gt.0)) THEN
                  CALL nesting (ng, inlm, nmflx)
                END IF
#    endif
              END DO
#   endif
!
!  Corrector AM3 substep with FB-feedback.
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (my_iif.le.nfast(ng)) THEN
                  knew(ng)=3-kstp(ng)
                  next_kstp(ng)=knew(ng)
 
                  DO tile=first_tile(ng),last_tile(ng),+1
                    CALL step2d (ng, tile)
                  END DO
!$OMP BARRIER
                END IF
              END DO
 
#   ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the state variables
!  associated with the 2D engine CORRECTOR STEP section (KNEW INDEX).
#    ifdef NESTING_DEBUG
!  If refinement, check mass flux conservation between coarse and
!  fine grids during debugging.
!  Warning: very verbose output to fort.300 ascii file to check
!           mass flux conservation.
#    endif
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (any(compositegrid(:,ng))) THEN
                  CALL nesting (ng, inlm, n2dcs)
                END IF
#    ifdef NESTING_DEBUG
                IF (refinedgrid(ng).and.(refinescale(ng).gt.0)) THEN
                  CALL nesting (ng, inlm, nmflx)
                END IF
#    endif
              END DO
#   endif
            END DO loop_2d
 
#  else
!
!-----------------------------------------------------------------------
!  Solve the vertically integrated primitive equations for the
!  free-surface and barotropic momentum components using a predictor-
!  corrector LeapFrog with 3rd-order Adams-Moulton (LF-AM3) time
!  stepping scheme.
!-----------------------------------------------------------------------
!
            loop_2d : DO my_iif=1,maxval(nfast)+1
!
!  Set time indices for predictor step. The PREDICTOR_2D_STEP switch
!  it is assumed to be false before the first time-step.
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                next_indx1=3-indx1(ng)
                IF (.not.predictor_2d_step(ng).and.                     &
     &              my_iif.le.(nfast(ng)+1)) THEN
                  predictor_2d_step(ng)=.true.
                  iif(ng)=my_iif
                  IF (first_2d_step) THEN
                    kstp(ng)=indx1(ng)
                  ELSE
                    kstp(ng)=3-indx1(ng)
                  END IF
                  knew(ng)=3
                  krhs(ng)=indx1(ng)
                END IF
!
!  Predictor step - Advance barotropic equations using 2D time-step
!  ==============   predictor scheme.  No actual time-stepping is
!  performed during the auxiliary (nfast+1) time-step. It is needed
!  to finalize the fast-time averaging of 2D fields, if any, and
!  compute the new time-evolving depths.
!
                IF (my_iif.le.(nfast(ng)+1)) THEN
                  DO tile=last_tile(ng),first_tile(ng),-1
                    CALL step2d (ng, tile)
                  END DO
!$OMP BARRIER
                END IF
              END DO
 
#   ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the state variables
!  associated with the 2D engine PREDICTOR STEP section.
#    ifdef NESTING_DEBUG
!  If refinement, check mass flux conservation between coarse and
!  fine grids during debugging.
!  Warning: very verbose output to fort.300 ascii file to check
!           mass flux conservation.
#    endif
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (any(compositegrid(:,ng))) THEN
                  CALL nesting (ng, inlm, n2dps)
                END IF
#    ifdef NESTING_DEBUG
                IF (refinedgrid(ng).and.(refinescale(ng).gt.0)) THEN
                  CALL nesting (ng, inlm, nmflx)
                END IF
#    endif
              END DO
#   endif
!
!  Set time indices for corrector step.
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (predictor_2d_step(ng)) THEN
                  predictor_2d_step(ng)=.false.
                  knew(ng)=next_indx1
                  kstp(ng)=3-knew(ng)
                  krhs(ng)=3
                  IF (iif(ng).lt.(nfast(ng)+1)) indx1(ng)=next_indx1
                END IF
!
!  Corrector step - Apply 2D time-step corrector scheme.  Notice that
!  ==============   there is not need for a corrector step during the
!  auxiliary (nfast+1) time-step.
!
                IF (iif(ng).lt.(nfast(ng)+1)) THEN
                  DO tile=first_tile(ng),last_tile(ng),+1
                    CALL step2d (ng, tile)
                  END DO
!$OMP BARRIER
                END IF
              END DO
 
#   ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the state variables
!  associated with the 2D engine CORRECTOR STEP section (KNEW INDEX).
#    ifdef NESTING_DEBUG
!  If refinement, check mass flux conservation between coarse and
!  fine grids during debugging.
!  Warning: very verbose output to fort.300 ascii file to check
!           mass flux conservation.
#    endif
!
              DO ig=1,gridsinlayer(nl)
                ng=gridnumber(ig,nl)
                IF (any(compositegrid(:,ng))) THEN
                  CALL nesting (ng, inlm, n2dcs)
                END IF
#    ifdef NESTING_DEBUG
                IF (refinedgrid(ng).and.(refinescale(ng).gt.0)) THEN
                  CALL nesting (ng, inlm, nmflx)
                END IF
#    endif
              END DO
#   endif
            END DO loop_2d
#  endif
 
# endif
 
# ifdef NESTING
#  if defined MASKING && defined WET_DRY
!
!-----------------------------------------------------------------------
!  If nesting and wetting and drying, scale horizontal interpolation
!  weights to account for land/sea masking in contact areas. This needs
!  to be done at very time-step since the Land/Sea masking is time
!  dependent.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              CALL nesting (ng, inlm, nmask)
            END DO
#  endif
!
!-----------------------------------------------------------------------
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for the time-averaged
!  momentum fluxes (DU_avg1, DV_avg1) and free-surface (Zt_avg).
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, n2dfx)
              END IF
            END DO
# endif
 
# if !(defined STEP2D_FB_AB3_AM4 || defined STEP2D_FB_LF_AM3)
!
!-----------------------------------------------------------------------
!  Recompute depths and thicknesses using the new time filtered
!  free-surface.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=last_tile(ng),first_tile(ng),-1
                CALL set_depth (ng, tile, inlm)
              END DO
!$OMP BARRIER
            END DO
# endif
 
# ifdef NESTING
!
!  If nesting, determine vertical indices and vertical interpolation
!  weights in the contact zone using new depth arrays.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
!             CALL nesting (ng, iNLM, nzwgt)
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Time-step 3D momentum equations.
!-----------------------------------------------------------------------
!
!  Time-step 3D momentum equations and couple with vertically
!  integrated equations.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=last_tile(ng),first_tile(ng),-1
                CALL step3d_uv (ng, tile)
              END DO
!$OMP BARRIER
            END DO
 
# ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for 3D momentum (u,v),
!  adjusted 2D momentum (ubar,vbar), and fluxes (Huon, Hvom).
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, n3duv)
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Time-step vertical mixing turbulent equations and passive tracer
!  source and sink terms, if applicable.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=first_tile(ng),last_tile(ng),+1
                CALL omega (ng, tile, inlm)
# ifdef MY25_MIXING
                CALL my25_corstep (ng, tile)
# elif defined GLS_MIXING
                CALL gls_corstep (ng, tile)
# endif
# if defined WEC_VF && defined WAVE_MIXING
                CALL wec_wave_mix (ng, tile)
# endif
# ifdef BIOLOGY
                CALL biology (ng, tile)
# endif
# ifdef SEDIMENT
                CALL sediment (ng, tile)
# endif
              END DO
!$OMP BARRIER
            END DO
 
# ifndef TS_FIXED
!
!-----------------------------------------------------------------------
!  Time-step tracer equations.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              DO tile=last_tile(ng),first_tile(ng),-1
                CALL step3d_t (ng, tile)
              END DO
!$OMP BARRIER
            END DO
 
#  ifdef NESTING
!
!  If composite or mosaic grids, process additional points in the
!  contact zone between connected grids for Tracer Variables.
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (any(compositegrid(:,ng))) THEN
                CALL nesting (ng, inlm, n3dtv)
              END IF
            END DO
#  endif
# endif
 
# ifdef NESTING
#  ifndef ONE_WAY
!
!-----------------------------------------------------------------------
!  If refinement grids, perform two-way coupling between fine and
!  coarse grids. Correct coarse grid tracers values at the refinement
!  grid with refined accumulated fluxes.  Then, replace coarse grid
!  state variable with averaged refined grid values (two-way nesting).
!  Update coarse grid depth variables.
!
!  The two-way exchange of infomation between nested grids needs to be
!  done at the correct time-step and in the right sequence.
!-----------------------------------------------------------------------
!
            DO il=nestlayers,1,-1
              DO ig=1,gridsinlayer(il)
                ng=gridnumber(ig,il)
                IF (do_twoway(inlm, nl, il, ng, istep)) THEN
                  CALL nesting (ng, inlm, n2way)
                END IF
              END DO
            END DO
#  endif
!
!-----------------------------------------------------------------------
!  If donor to a finer grid, extract data for the external contact
!  points. This is the latest solution for the coarser grid.
!
!  It is stored in the REFINED structure so it can be used for the
!  space-time interpolation when "nputD" argument is used above.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (donortofiner(ng)) THEN
                CALL nesting (ng, inlm, ngetd)
              END IF
            END DO
# endif
 
# ifdef FLOATS
!
!-----------------------------------------------------------------------
!  Compute Lagrangian drifters trajectories: Split all the drifters
!  between all the computational threads, except in distributed-memory
!  and serial configurations. In distributed-memory, the parallel node
!  containing the drifter is selected internally since the state
!  variables do not have a global scope.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              IF (lfloats(ng)) THEN
#  ifdef _OPENMP
                chunk_size=(nfloats(ng)+numthreads-1)/numthreads
                lstr=1+mythread*chunk_size
                lend=min(nfloats(ng),lstr+chunk_size-1)
#  else
                lstr=1
                lend=nfloats(ng)
#  endif
                CALL step_floats (ng, lstr, lend)
!$OMP BARRIER
!
!  Shift floats time indices.
!
                nfp1(ng)=mod(nfp1(ng)+1,nft+1)
                nf(ng)=mod(nf(ng)+1,nft+1)
                nfm1(ng)=mod(nfm1(ng)+1,nft+1)
                nfm2(ng)=mod(nfm2(ng)+1,nft+1)
                nfm3(ng)=mod(nfm3(ng)+1,nft+1)
              END IF
            END DO
# endif
!
!-----------------------------------------------------------------------
!  Advance time index and time clock.
!-----------------------------------------------------------------------
!
            DO ig=1,gridsinlayer(nl)
              ng=gridnumber(ig,nl)
              iic(ng)=iic(ng)+1
              time(ng)=time(ng)+dt(ng)
              step_counter(ng)=step_counter(ng)-1
              CALL time_string (time(ng), time_code(ng))
            END DO
 
          END DO step_loop
 
        END DO nest_layer
 
      END DO kernel_loop
 
      RETURN
      END SUBROUTINE main3d
#else
      SUBROUTINE main3d
      RETURN
      END SUBROUTINE main3d
#endif