tlcheck_roms.h

Go to the documentation of this file.

      MODULE roms_kernel_mod
!
!git $Id$
!================================================== Hernan G. Arango ===
!  Copyright (c) 2002-2025 The ROMS Group            Andrew M. Moore   !
!    Licensed under a MIT/X style license                              !
!    See License_ROMS.md                                               !
!=======================================================================
!                                                                      !
!  ROMS Tangent Linear Model Linearization Test:                       !
!                                                                      !
!  This driver is used to check the "linearization" of the tangent     !
!  linear model using a structure similar to the gradient test.        !
!                                                                      !
!  The subroutines in this driver control the initialization, time-    !
!  stepping,  and  finalization of ROMS  model following ESMF          !
!  conventions:                                                        !
!                                                                      !
!     ROMS_initialize                                                  !
!     ROMS_run                                                         !
!     ROMS_finalize                                                    !
!                                                                      !
!=======================================================================
!
      USE mod_param
      USE mod_parallel
      USE mod_arrays
      USE mod_fourdvar
      USE mod_iounits
      USE mod_ncparam
      USE mod_scalars
      USE mod_stepping
!
      USE close_io_mod,      ONLY : close_file, close_inp, close_out
      USE def_mod_mod,       ONLY : def_mod
      USE dotproduct_mod,    ONLY : ad_dotproduct
      USE get_state_mod,     ONLY : get_state
      USE inp_par_mod,       ONLY : inp_par
#ifdef MCT_LIB
# ifdef ATM_COUPLING
      USE mct_coupler_mod,   ONLY : initialize_ocn2atm_coupling
# endif
# ifdef WAV_COUPLING
      USE mct_coupler_mod,   ONLY : initialize_ocn2wav_coupling
# endif
#endif
      USE stdout_mod,        ONLY : set_stdoutunit, stdout_unit
      USE strings_mod,       ONLY : founderror
      USE wrt_rst_mod,       ONLY : wrt_rst
!
      implicit none
!
      PUBLIC :: roms_initialize
      PUBLIC :: roms_run
      PUBLIC :: roms_finalize
!
      CONTAINS
!
      SUBROUTINE roms_initialize (first, mpiCOMM)
!
!=======================================================================
!                                                                      !
!  This routine allocates and initializes ROMS state variables         !
!  and internal and external parameters.                               !
!                                                                      !
!=======================================================================
!
!  Imported variable declarations.
!
      logical, intent(inout) :: first
!
      integer, intent(in), optional :: mpiCOMM
!
!  Local variable declarations.
!
      logical :: allocate_vars = .true.
!
#ifdef DISTRIBUTE
      integer :: MyError, MySize
#endif
      integer :: chunk_size, ng, thread
!
      character (len=*), parameter :: MyFile =                          &
     &  __FILE__//", ROMS_initialize"
 
#ifdef DISTRIBUTE
!
!-----------------------------------------------------------------------
!  Set distribute-memory (mpi) world communictor.
!-----------------------------------------------------------------------
!
      IF (PRESENT(mpicomm)) THEN
        ocn_comm_world=mpicomm
      ELSE
        ocn_comm_world=mpi_comm_world
      END IF
      CALL mpi_comm_rank (ocn_comm_world, myrank, myerror)
      CALL mpi_comm_size (ocn_comm_world, mysize, myerror)
#endif
!
!-----------------------------------------------------------------------
!  On first pass, initialize model parameters a variables for all
!  nested/composed grids.  Notice that the logical switch "first"
!  is used to allow multiple calls to this routine during ensemble
!  configurations.
!-----------------------------------------------------------------------
!
      IF (first) THEN
        first=.false.
!
!  Initialize parallel control switches. These scalars switches are
!  independent from standard input parameters.
!
        CALL initialize_parallel
!
!  Set the ROMS standard output unit to write verbose execution info.
!  Notice that the default standard out unit in Fortran is 6.
!
!  In some applications like coupling or disjointed mpi-communications,
!  it is advantageous to write standard output to a specific filename
!  instead of the default Fortran standard output unit 6. If that is
!  the case, it opens such formatted file for writing.
!
        IF (set_stdoutunit) THEN
          stdout=stdout_unit(master)
          set_stdoutunit=.false.
        END IF
!
!  Read in model tunable parameters from standard input. Allocate and
!  initialize variables in several modules after the number of nested
!  grids and dimension parameters are known.
!
        CALL inp_par (inlm)
        IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
!
!  Set domain decomposition tile partition range.  This range is
!  computed only once since the "first_tile" and "last_tile" values
!  are private for each parallel thread/node.
!
#if defined _OPENMP
      mythread=my_threadnum()
#elif defined DISTRIBUTE
      mythread=myrank
#else
      mythread=0
#endif
      DO ng=1,ngrids
        chunk_size=(ntilex(ng)*ntilee(ng)+numthreads-1)/numthreads
        first_tile(ng)=mythread*chunk_size
        last_tile(ng)=first_tile(ng)+chunk_size-1
      END DO
!
!  Initialize internal wall clocks. Notice that the timings does not
!  includes processing standard input because several parameters are
!  needed to allocate clock variables.
!
        IF (master) THEN
          WRITE (stdout,10)
 10       FORMAT (/,' Process Information:',/)
        END IF
!
        DO ng=1,ngrids
          DO thread=thread_range
            CALL wclock_on (ng, inlm, 0, __line__, myfile)
          END DO
        END DO
!
!  Allocate and initialize modules variables.
!
        CALL roms_allocate_arrays (allocate_vars)
        CALL roms_initialize_arrays
        IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
      END IF
 
#if defined MCT_LIB && (defined ATM_COUPLING || defined WAV_COUPLING)
!
!-----------------------------------------------------------------------
!  Initialize coupling streams between model(s).
!-----------------------------------------------------------------------
!
      DO ng=1,ngrids
# ifdef ATM_COUPLING
        CALL initialize_ocn2atm_coupling (ng, myrank)
# endif
# ifdef WAV_COUPLING
        CALL initialize_ocn2wav_coupling (ng, myrank)
# endif
      END DO
#endif
!
      RETURN
      END SUBROUTINE roms_initialize
!
      SUBROUTINE roms_run (RunInterval)
!
!=======================================================================
!                                                                      !
!  This routine time-steps ROMS nonlinear, tangent linear and          !
!  adjoint models.                                                     !
!                                                                      !
!=======================================================================
!
!  Imported variable declarations
!
      real(dp), intent(in) :: RunInterval            ! seconds
!
!  Local variable declarations.
!
      integer :: IniRec, Ipass, i, ng, status
      integer :: tile
!
      real(r8) :: gp, hp, p
!
      character (len=*), parameter :: MyFile =                          &
     &  __FILE__//", ROMS_run"
!
!-----------------------------------------------------------------------
!  Run tangent linear model test.
!-----------------------------------------------------------------------
!
!  Currently, the tangent linear model test cannot be run over
!  nested grids.
!
      IF (ngrids.gt.1) THEN
        WRITE (stdout,10) 'Nested grids are not allowed, Ngrids = ',    &
                          ngrids
        stop
      END IF
!
!  Initialize relevant parameters.
!
      DO ng=1,ngrids
        lold(ng)=1
        lnew(ng)=1
        ntlm(ng)=nhis(ng)                      ! to allow IO comparison
#ifdef FORWARD_FLUXES
        lreadblk(ng)=.false.
#endif
        lreadfwd(ng)=.false.
      END DO
      nrun=1
      ipass=1
      erstr=1
      erend=maxval(nstatevar)+1
      inirec=1
      ig1count=0
      ig2count=0
      lcycletlm=.false.
!
!  Initialize nonlinear model with first guess initial conditions.
!
      CALL initial
      IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
!
!  Define output 4D-Var NetCDF file (DAV struc) containing all
!  processed data at observation locations.
!
      DO ng=1,ngrids
        ldefmod(ng)=.true.
        CALL def_mod (ng)
        IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
      END DO
!
!  Run nonlinear model. Extract and store nonlinear model values at
!  observation locations.
!
      DO ng=1,ngrids
        IF (master) THEN
          WRITE (stdout,20) 'NL', ng, ntstart(ng), ntend(ng)
        END IF
        wrtnlmod(ng)=.true.
        wrttlmod(ng)=.false.
      END DO
!
#ifdef SOLVE3D
      CALL main3d (runinterval)
#else
      CALL main2d (runinterval)
#endif
      IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
      DO ng=1,ngrids
        wrtnlmod(ng)=.false.
        wrttlmod(ng)=.true.
      END DO
!
!  Set structure for the nonlinear forward trajectory to be processed
!  by the tangent linear and adjoint models. Also, set switches to
!  process the FWD structure in routine "check_multifile". Notice that
!  it is possible to split solution into multiple NetCDF files to reduce
!  their size.
!
      CALL edit_multifile ('HIS2FWD')
      IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
      DO ng=1,ngrids
        lreadfwd(ng)=.true.
      END DO
 
#ifdef FORWARD_FLUXES
!
!  Set structure for the nonlinear surface fluxes to be processed by
!  by the tangent linear and adjoint models. Also, set switches to
!  process the BLK structure in routine "check_multifile".  Notice that
!  it is possible to split solution into multiple NetCDF files to reduce
!  their size.
!
!  The switch LreadFRC is deactivated because all the atmospheric
!  forcing, including shortwave radiation, is read from the NLM bulk
!  flux formulation or is assigned during ESM coupling (BLK structure).
!  There is no need for processing forcing from the FRC structure files.
!
      CALL edit_multifile ('QCK2BLK')
      IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
      DO ng=1,ngrids
        lreadblk(ng)=.true.
        lreadfrc(ng)=.false.
      END DO
#endif
!
!  Save and Report cost function between nonlinear model and
!  observations.
!
      DO ng=1,ngrids
        DO i=0,nobsvar(ng)
          fourdvar(ng)%CostFunOld(i)=fourdvar(ng)%CostFun(i)
        END DO
        IF (master) THEN
          WRITE (stdout,40) fourdvar(ng)%CostFunOld(0)
          DO i=1,nobsvar(ng)
            IF (fourdvar(ng)%CostFunOld(i).gt.0.0_r8) THEN
              IF (i.eq.1) THEN
                WRITE (stdout,50) fourdvar(ng)%CostFunOld(i),           &
     &                            trim(obsname(i))
              ELSE
                WRITE (stdout,60) fourdvar(ng)%CostFunOld(i),           &
     &                            trim(obsname(i))
              END IF
            END IF
          END DO
        END IF
      END DO
!
!  Initialize the adjoint model from rest.
!
      DO ng=1,ngrids
!!      CALL ad_initial (ng, .TRUE.)
        CALL ad_initial (ng)
        IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
      END DO
!
!  Time-step adjoint model: Compute model state gradient, GRAD(J).
!  Force the adjoint model with the adjoint misfit between nonlinear
!  model and observations.
!
      DO ng=1,ngrids
        IF (master) THEN
          WRITE (stdout,20) 'AD', ng, ntstart(ng), ntend(ng)
        END IF
      END DO
!
#ifdef SOLVE3D
      CALL ad_main3d (runinterval)
#else
      CALL ad_main2d (runinterval)
#endif
      IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
!
!-----------------------------------------------------------------------
!  Perturb each tangent linear state variable using the steepest decent
!  direction by grad(J). If ndefTLM < 0, suppress IO for both nonlinear
!  and tangent linear models in the outer and inner loops.
!-----------------------------------------------------------------------
!
!  Load adjoint solution.
!
      DO ng=1,ngrids
        CALL get_state (ng, iadm, 3, adm(ng), adm(ng)%Rindex, lnew(ng))
        IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
      END DO
!
!  Compute adjoint solution dot product for scaling purposes.
!
      DO ng=1,ngrids
        DO tile=first_tile(ng),last_tile(ng),+1
          CALL ad_dotproduct (ng, tile, lnew(ng))
        END DO
      END DO
 
#ifdef SOLVE3D
!
!  OUTER LOOP: First, Perturb all state variables (zeta, u, v, t) at
!  ==========  once. Then, perturb one state variable at the time.
!              Notice, that ubar and vbar are not perturbed. They are
!              computed by vertically integarting u and v.
!
#else
!
!  OUTER LOOP: First, perturb all state variables (zeta, ubar, vbar) at
!  ==========  once. Then, perturb one state variable at the time.
!
#endif
      outer_loop : DO outer=erstr,erend
 
        DO ng=1,ngrids
          CALL get_state (ng, inlm, 1, fwd(ng), inirec, lnew(ng))
          IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
        END DO
!
!  INNER LOOP: scale perturbation amplitude by selecting "p" scalar,
!  ==========  such that:
!                              p = 10 ** REAL(-inner,r8)
!
        inner_loop : DO inner=1,ninner
!
!  Add a perturbation to the nonlinear state initial conditions
!  according with the outer and inner loop iterations. The added
!  term is a function of the steepest descent direction defined
!  by grad(J) times the perturbation amplitude "p".
!
!!        CALL initial (.FALSE.)
          CALL initial
          IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
          DO ng=1,ngrids
            WRITE (his(ng)%name,70) trim(his(ng)%base), nrun
 
            IF (ndeftlm(ng).lt.0) THEN
              ldefhis(ng)=.false.              ! suppress IO
              lwrthis(ng)=.false.
            ELSE
              ldefhis(ng)=.true.
            END IF
            wrtnlmod(ng)=.true.
            wrttlmod(ng)=.false.
          END DO
!
!  Time-step nonlinear model: compute perturbed nonlinear state.
!
          DO ng=1,ngrids
            IF (master) THEN
              WRITE (stdout,20) 'NL', ng, ntstart(ng), ntend(ng)
            END IF
          END DO
!
#ifdef SOLVE3D
          CALL main3d (runinterval)
#else
          CALL main2d (runinterval)
#endif
          IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
!
!  Get current nonlinear model trajectory.
!
          DO ng=1,ngrids
            fwd(ng)%name=trim(his(ng)%head)//'.nc'
            CALL get_state (ng, inlm, 1, fwd(ng), inirec, lnew(ng))
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
          END DO
!
!  Initialize tangent linear with the steepest decent direction
!  (adjoint state, GRAD(J)) times the perturbation amplitude "p".
!
          DO ng=1,ngrids
!!          CALL tl_initial (ng, .FALSE.)
            CALL tl_initial (ng)
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
 
            WRITE (tlm(ng)%name,70) trim(tlm(ng)%base), nrun
 
            IF (ndeftlm(ng).lt.0) THEN
              ldeftlm(ng)=.false.              ! suppress IO
              lwrttlm(ng)=.false.
            ELSE
              ldeftlm(ng)=.true.
            END IF
          END DO
!
!  Time-step tangent linear model:  Compute misfit cost function
!  between model (nonlinear + tangent linear) and observations.
!
          DO ng=1,ngrids
            IF (master) THEN
              WRITE (stdout,20) 'TL', ng, ntstart(ng), ntend(ng)
            END IF
          END DO
!
#ifdef SOLVE3D
          CALL tl_main3d (runinterval)
#else
          CALL tl_main2d (runinterval)
#endif
          IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
!
!  Close current tangent linear model history file.
!
          sourcefile=myfile
          DO ng=1,ngrids
            CALL close_file (ng, itlm, tlm(ng))
            IF (founderror(exit_flag, noerror, __line__, myfile)) RETURN
          END DO
!
!  Advance model run counter.
!
          nrun=nrun+1
 
        END DO inner_loop
 
      END DO outer_loop
!
!  Report dot products.
!
      DO ng=1,ngrids
        IF (master) THEN
          WRITE (stdout,80)                                             &
     &      'TLM Test - Dot Products Summary: p, g1, g2, (g1-g2)/g1'
          inner=1
          DO i=1,min(ig1count,ig2count)
            p=10.0_r8**real(-inner,r8)
            IF (mod(i,1+ntimes(ng)/ntlm(ng)).eq.0) inner=inner+1
            WRITE (stdout,90) i, p, g1(i), g2(i), (g1(i)-g2(i))/g1(i)
            IF ((mod(i,1+ntimes(ng)/ntlm(ng)).eq.0).and.                &
     &          (mod(i,ninner*(1+ntimes(ng)/ntlm(ng))).ne.0)) THEN
              WRITE (stdout,100)
            ELSE IF (mod(i,ninner*(1+ntimes(ng)/ntlm(ng))).eq.0) THEN
              inner=1
              WRITE (stdout,110)
            END IF
          END DO
        END IF
      END DO
!
 10   FORMAT (/,a,i3,/)
 20   FORMAT (/,1x,a,1x,'ROMS : started time-stepping:',                &
     &        ' (Grid: ',i2.2,' TimeSteps: ',i8.8,' - ',i8.8,')',/)
 40   FORMAT (/,' Nonlinear Model Cost Function = ',1p,e21.14)
 50   FORMAT (' --------------- ','cost function = ',1p,e21.14,2x,a)
 60   FORMAT (17x,'cost function = ',1p,e21.14,2x,a)
 70   FORMAT (a,'_',i3.3,'.nc')
 80   FORMAT (/,a,/)
 90   FORMAT (i4,2x,1pe8.1,3(1x,1p,e20.12,0p))
100   FORMAT (77('.'))
110   FORMAT (77('-'))
!
      RETURN
      END SUBROUTINE roms_run
!
      SUBROUTINE roms_finalize
!
!=======================================================================
!                                                                      !
!  This routine terminates ROMS nonlinear, tangent linear, and         !
!  adjoint models execution.                                           !
!                                                                      !
!=======================================================================
!
!  Local variable declarations.
!
      integer :: Fcount, ng, thread
!
      character (len=*), parameter :: MyFile =                          &
     &  __FILE__//", ROMS_finalize"
!
!-----------------------------------------------------------------------
!  If blowing-up, save latest model state into RESTART NetCDF file.
!-----------------------------------------------------------------------
!
!  If cycling restart records, write solution into the next record.
!
      IF (exit_flag.eq.1) THEN
        DO ng=1,ngrids
          IF (lwrtrst(ng)) THEN
            IF (master) WRITE (stdout,10)
 10         FORMAT (/,' Blowing-up: Saving latest model state into ',   &
     &                ' RESTART file',/)
            fcount=rst(ng)%load
            IF (lcyclerst(ng).and.(rst(ng)%Nrec(fcount).ge.2)) THEN
              rst(ng)%Rindex=2
              lcyclerst(ng)=.false.
            END IF
            blowup=exit_flag
            exit_flag=noerror
#ifdef DISTRIBUTE
            CALL wrt_rst (ng, myrank)
#else
            CALL wrt_rst (ng, -1)
#endif
          END IF
        END DO
      END IF
!
!-----------------------------------------------------------------------
!  Stop model and time profiling clocks, report memory requirements, and
!  close output NetCDF files.
!-----------------------------------------------------------------------
!
!  Stop time clocks.
!
      IF (master) THEN
        WRITE (stdout,20)
 20     FORMAT (/,'Elapsed wall CPU time for each process (seconds):',/)
      END IF
!
      DO ng=1,ngrids
        DO thread=thread_range
          CALL wclock_off (ng, inlm, 0, __line__, myfile)
        END DO
      END DO
!
!  Report dynamic memory and automatic memory requirements.
!
      CALL memory
!
!  Close IO files.
!
      DO ng=1,ngrids
        CALL close_inp (ng, inlm)
      END DO
      CALL close_out
!
      RETURN
      END SUBROUTINE roms_finalize
 
      END MODULE roms_kernel_mod