timers.F File Reference

#include "cppdefs.h"

Include dependency graph for timers.F:

Go to the source code of this file.

Functions/Subroutines
recursive subroutine	wclock_on (ng, model, region, line, routine)
recursive subroutine	wclock_off (ng, model, region, line, routine)

Function/Subroutine Documentation

wclock_off()

recursive subroutine wclock_off	(	integer, intent(in)	ng,
		integer, intent(in)	model,
		integer, intent(in)	region,
		integer, intent(in)	line,
		character (len=*), intent(in)	routine )

Definition at line 147 of file timers.F.

!
!=======================================================================
!                                                                      !
!  This routine turns off wall clock to meassure the elapsed time in   !
!  seconds spend by each parallel thread in requested model region.    !
!                                                                      !
!  On Input:                                                           !
!                                                                      !
!     ng         Nested grid number (integer)                          !
!     model      Calling model identifier (integer)                    !
!     region     Profiling region number (integer)                     !
!     line       Calling model routine line (integer)                  !
!     routine    Calling model routine (string)                        !
!                                                                      !
!=======================================================================
!
      USE mod_param
      USE mod_parallel
      USE mod_iounits
      USE mod_strings
 
#ifdef DISTRIBUTE
!
      USE distribute_mod, ONLY : mp_barrier, mp_reduce
      USE distribute_mod, ONLY : mp_collect
      USE strings_mod,    ONLY : uppercase
#endif
!
      implicit none
!
!  Imported variable declarations.
!
      integer, intent(in) ::  ng, model, region, line
 
      character (len=*), intent(in) :: routine
!
!  Local variable declarations.
!
      integer :: ig, imodel, iregion, MyModel, NSUB
#ifdef DISTRIBUTE
      integer :: MyCOMM, nPETs, PETrank
# ifdef ROMS_STDOUT
      integer :: node
# endif
#endif
      integer :: my_threadnum
 
      real(r8) :: percent, sumcpu, sumper, sumsum, total
 
      real(r8), dimension(2) :: wtime
 
      real(r8) :: my_wtime
 
#ifdef DISTRIBUTE
      real(r8) :: TendMin, TendMax
 
      real(r8), parameter :: Tspv = 0.0_r8
      real(r8), allocatable :: Tend(:)
      real(r8), dimension(0:Nregion) :: rbuffer
 
      character (len= 3), dimension(0:Nregion) :: op_handle
#endif
      character (len=14), dimension(4) :: label
!
!-----------------------------------------------------------------------
!  Compute elapsed wall time for all threads.
!-----------------------------------------------------------------------
#ifdef TRACING
!
      IF (master) THEN
        WRITE (stdout,'(a,a)') '<== Exiting  ', trim(routine)
        FLUSH (stdout)
      END IF
#endif
!
!  Set number of subdivisions, same as for global reductions.
!
#ifdef DISTRIBUTE
# ifdef DISJOINTED
      mycomm=full_comm_world
      npets=fullsize
      petrank=fullrank
# else
      mycomm=ocn_comm_world
      npets=numthreads
      petrank=myrank
# endif
      nsub=1
#else
      nsub=numthreads
#endif
!
!  Insure that MyModel is not zero.
!
      mymodel=max(1,model)
!
!  Compute elapsed CPU time (seconds) for each profile region, except
!  for region zero which is called by the main driver before the
!  simulatiom is stopped.
!
      IF (region.ne.0) THEN
        cend(region,mymodel,ng)=cend(region,mymodel,ng)+                &
     &                          (my_wtime(wtime)-                       &
     &                           cstr(region,mymodel,ng))
      END IF
!
!-----------------------------------------------------------------------
!  If simulation is compleated, compute and report elapsed CPU time for
!  all regions.
!-----------------------------------------------------------------------
!
      IF ((region.eq.0).and.(proc(1,mymodel,ng).eq.1)) THEN
!
!  Computed elapsed wall time for the driver, region=0.  Since it is
!  called only once, "MyModel" will have a value and the other models
!  will be zero.
!
        cend(region,mymodel,ng)=cend(region,mymodel,ng)+                &
     &                          (my_wtime(wtime)-                       &
     &                           cstr(region,mymodel,ng))
        DO imodel=1,4
          proc(1,imodel,ng)=0
        END DO
 
!$OMP CRITICAL (FINALIZE_WCLOCK)
!
!  Report elapsed time (seconds) for each CPU.  We get the same time
!  time for all nested grids.
!
        IF (ng.eq.1) THEN
#ifdef DISTRIBUTE
          CALL mp_barrier (ng, model, mycomm)
          IF (.not.allocated(tend)) THEN
            allocate ( tend(npets) )
            tend=0.0_r8
          END IF
          tend(petrank+1)=cend(region,mymodel,ng)
          CALL mp_collect (ng, model, npets, tspv, tend, mycomm)
          tendmin=minval(tend)
          tendmax=maxval(tend)
# ifdef ROMS_STDOUT
          IF (master) THEN
            DO node=1,npets
               WRITE (stdout,10) ' Node   #', node-1,                   &
     &                           ' CPU:', tend(node)
            END DO
          END IF
# else
          WRITE (stdout,10) ' Node   #', petrank,                       &
     &                      ' CPU:', tend(petrank+1)
# endif
          FLUSH (stdout)
#else
          WRITE (stdout,10) ' Thread #', mythread, ' CPU:',             &
     &                      cend(region,mymodel,ng)
#endif
 10       FORMAT (a,i5,a,f12.3)
        END IF
!
!  Sum the elapsed time for each profile region by model.
!
        thread_count=thread_count+1
        DO imodel=1,4
          DO iregion=0,nregion
            csum(iregion,imodel,ng)=csum(iregion,imodel,ng)+            &
     &                              cend(iregion,imodel,ng)
          END DO
        END DO
!
!  Compute total elapsed CPU wall time between all parallel processes.
!
        IF (thread_count.eq.nsub) THEN
          thread_count=0
#ifdef DISTRIBUTE
          op_handle(0:nregion)='SUM'      ! Gather all values using a
          DO imodel=1,4                   ! reduced sum between nodes
            DO iregion=0,nregion
              rbuffer(iregion)=csum(iregion,imodel,ng)
            END DO
            CALL mp_reduce (ng, mymodel, nregion+1, rbuffer(0:),        &
     &                      op_handle(0:), mycomm)
            DO iregion=0,nregion
              csum(iregion,imodel,ng)=rbuffer(iregion)
            END DO
          END DO
#endif
          IF (master) THEN
            IF (ng.eq.1) THEN             ! Same for all nested grids
              total_cpu=total_cpu+csum(region,model,ng)
            END IF
            DO imodel=1,4
              total_model(imodel)=0.0_r8
              DO iregion=1,nregion
                total_model(imodel)=total_model(imodel)+                &
     &                              csum(iregion,imodel,ng)
              END DO
            END DO
            IF (ng.eq.1) THEN
              WRITE (stdout,20) ' Total:', total_cpu
 20           FORMAT (a,t18,f14.3)
              IF (numthreads.gt.1) THEN
                WRITE (stdout,20) ' Average:', total_cpu/numthreads
#ifdef DISTRIBUTE
                WRITE (stdout,20) ' Minimum:', tendmin
                WRITE (stdout,20) ' Maximum:', tendmax
#endif
              END IF
            END IF
          END IF
 
#ifdef DISTRIBUTE
          IF (allocated(tend)) deallocate (tend)
#endif
 
#ifdef PROFILE
!
!  Report profiling times.
!
          label(inlm)='Nonlinear     '
          label(itlm)='Tangent linear'
          label(irpm)='Representer   '
          label(iadm)='Adjoint       '
          DO imodel=1,4
            IF (master.and.(total_model(imodel).gt.0.0_r8)) THEN
              WRITE (stdout,30) trim(label(imodel)),                    &
     &                          'model elapsed CPU time profile, Grid:',&
     &                          ng
 30           FORMAT (/,1x,a,1x,a,1x,i2.2/)
            END IF
            sumcpu=0.0_r8
            sumper=0.0_r8
            DO iregion=1,mregion-1
              IF (master.and.(csum(iregion,imodel,ng).gt.0.0_r8)) THEN
                percent=100.0_r8*csum(iregion, imodel,ng)/total_cpu
                WRITE (stdout,40) pregion(iregion),                     &
     &                            csum(iregion,imodel,ng), percent
                sumcpu=sumcpu+csum(iregion,imodel,ng)
                sumper=sumper+percent
              END IF
            END DO
            ctotal=ctotal+sumcpu
 40         FORMAT (2x,a,t53,f14.3,2x,'(',f7.4,' %)')
            IF (master.and.(total_model(imodel).gt.0.0_r8)) THEN
              WRITE (stdout,50) sumcpu, sumper
 50           FORMAT (t47,'Total:',f14.3,2x,f8.4,' %')
            END IF
          END DO
!
!  Sometimes the profiling does not fully accounts for all the CPU
!  spend outside of the ROMS kernels. In data assimilation algorithms,
!  there is a lot of CPU expend outside the kernels. A separated
!  profiling is reported bellow.
!
          IF (master.and.(ng.eq.ngrids)) THEN
            percent=100.0_r8*ctotal/total_cpu
            WRITE (stdout,60) ctotal, percent,                          &
     &                        total_cpu-ctotal, 100.0_r8-percent
 60         FORMAT (/,2x,                                               &
     &             'Unique kernel(s) regions profiled ................',&
     &               f14.3,2x,f8.4,' %'/,2x,                            &
     &             'Residual, non-profiled code ......................',&
     &               f14.3,2x,f8.4,' %'/)
            WRITE (stdout,70) total_cpu
 70         FORMAT (/,' All percentages are with respect to',           &
     &                ' total time =',5x,f12.3,/)
          END IF
 
# ifdef FOUR_DVAR
!
!  Report elapsed time for 4D-Var algorithms.
!
          total=0.0_r8
          DO iregion=fregion,nregion-3
            DO imodel=1,4
              total=total+csum(iregion,imodel,ng)
            END DO
          END DO
          IF (master.and.(total.gt.0.0_r8)) THEN
            WRITE (stdout,30) 'Variational',                            &
     &                        'data assimilation profile, Grid:', ng
          END IF
          IF (total.gt.0.0_r8) THEN
            sumper=0.0_r8
            sumsum=0.0_r8
            DO iregion=fregion,nregion-3
              sumcpu=0.0_r8
              DO imodel=1,4
                sumcpu=sumcpu+csum(iregion,imodel,ng)
              END DO
              IF (master.and.(sumcpu.gt.0.0_r8)) THEN
                percent=100.0_r8*sumcpu/total_cpu
                WRITE (stdout,40) pregion(iregion), sumcpu, percent
                sumsum=sumsum+sumcpu
                sumper=sumper+percent
              END IF
            END DO
            IF (master.and.(total.gt.0.0_r8)) THEN
              WRITE (stdout,80) sumsum
 80           FORMAT (t47,'Total:',f14.3)
            END IF
          END IF
!
!  The background, increment, and analysis should be close to 100
!  percent.  The are listed last in Pregion.
!
          total=0.0_r8
          DO iregion=nregion-2,nregion
            DO imodel=1,4
              total=total+csum(iregion,imodel,ng)
            END DO
          END DO
          IF (master.and.(total.gt.0.0_r8)) THEN
            WRITE (stdout,'(1x)')
          END IF
          IF (total.gt.0.0_r8) THEN
            sumper=0.0_r8
            sumsum=0.0_r8
            DO iregion=nregion-2,nregion
              sumcpu=0.0_r8
              DO imodel=1,4
                sumcpu=sumcpu+csum(iregion,imodel,ng)
              END DO
              IF (master.and.(sumcpu.gt.0.0_r8)) THEN
                percent=100.0_r8*sumcpu/total_cpu
                WRITE (stdout,40) pregion(iregion), sumcpu, percent
                sumsum=sumsum+sumcpu
                sumper=sumper+percent
              END IF
            END DO
            IF (master.and.(total.gt.0.0_r8)) THEN
              WRITE (stdout,50) sumsum, sumper
            END IF
          END IF
# endif
# ifdef DISTRIBUTE
!
!  Report elapsed time for message passage communications.
!
          total=0.0_r8
          DO iregion=mregion,fregion-1
            DO imodel=1,4
              total=total+csum(iregion,imodel,ng)
            END DO
          END DO
          IF (master.and.(total.gt.0.0_r8)) THEN
            WRITE (stdout,30) uppercase('mpi'),                         &
     &                        'communications profile, Grid:', ng
          END IF
          IF (total.gt.0.0_r8) THEN
            sumper=0.0_r8
            sumsum=0.0_r8
            DO iregion=mregion,fregion-1
              sumcpu=0.0_r8
              DO imodel=1,4
                sumcpu=sumcpu+csum(iregion,imodel,ng)
              END DO
              IF (master.and.(sumcpu.gt.0.0_r8)) THEN
                percent=100.0_r8*sumcpu/total_cpu
                WRITE (stdout,40) pregion(iregion), sumcpu, percent
                sumsum=sumsum+sumcpu
                sumper=sumper+percent
              END IF
            END DO
            IF (master.and.(total.gt.0.0_r8)) THEN
              WRITE (stdout,50) sumsum, sumper
            END IF
          END IF
 
#  ifdef NESTING
!
!  Report total elapsed time for message passage communications for
!  all nested grids.
!
          IF (ng.eq.ngrids) THEN
            total=0.0_r8
            DO ig=1,ngrids
              DO iregion=mregion,fregion-1
                DO imodel=1,4
                  total=total+csum(iregion,imodel,ig)
                END DO
              END DO
            END DO
            IF (master.and.(total.gt.0.0_r8)) THEN
               WRITE (stdout,90) uppercase('mpi'),                      &
     &                'communications profile over all nested grids:'
 90            FORMAT (/,1x,a,1x,a,/)
            END IF
            IF (total.gt.0.0_r8) THEN
              sumper=0.0_r8
              sumsum=0.0_r8
              DO iregion=mregion,fregion-1
                sumcpu=0.0_r8
                DO ig=1,ngrids
                  DO imodel=1,4
                    sumcpu=sumcpu+csum(iregion,imodel,ig)
                  END DO
                END DO
                IF (master.and.(sumcpu.gt.0.0_r8)) THEN
                  percent=100.0_r8*sumcpu/total_cpu
                  WRITE (stdout,40) pregion(iregion), sumcpu, percent
                  sumsum=sumsum+sumcpu
                  sumper=sumper+percent
                END IF
              END DO
              IF (master.and.(total.gt.0.0_r8)) THEN
                WRITE (stdout,50) sumsum, sumper
              END IF
            END IF
          END IF
#  endif
# endif
#endif
        END IF
!$OMP END CRITICAL (FINALIZE_WCLOCK)
      END IF
      RETURN

References mod_parallel::cend, mod_parallel::cstr, mod_parallel::csum, mod_parallel::ctotal, mod_strings::fregion, mod_parallel::fullrank, mod_param::iadm, mod_param::inlm, mod_param::irpm, mod_param::itlm, mod_parallel::master, distribute_mod::mp_barrier(), mod_strings::mregion, my_wtime(), mod_parallel::myrank, mod_parallel::mythread, mod_param::ngrids, mod_strings::nregion, mod_parallel::numthreads, mod_parallel::ocn_comm_world, mod_strings::pregion, mod_parallel::proc, mod_iounits::stdout, mod_parallel::thread_count, mod_parallel::total_cpu, mod_parallel::total_model, and strings_mod::uppercase().

Referenced by ad_biology_mod::ad_biology(), ad_bulk_flux_mod::ad_bulk_flux(), ad_diag_mod::ad_diag(), dotproduct_mod::ad_dotproduct(), ad_frc_adjust_mod::ad_frc_adjust(), ad_get_data(), ad_get_idata(), ad_ini_fields_mod::ad_ini_fields(), ini_adjust_mod::ad_ini_perturb(), ad_ini_fields_mod::ad_ini_zeta(), ad_initial(), inner2state_mod::ad_inner2state(), mp_exchange_mod::ad_mp_exchange2d(), mp_exchange_mod::ad_mp_exchange2d_bry(), mp_exchange_mod::ad_mp_exchange3d(), mp_exchange_mod::ad_mp_exchange3d_bry(), mp_exchange_mod::ad_mp_exchange4d(), ad_obc_adjust_mod::ad_obc2d_adjust(), ad_obc_adjust_mod::ad_obc_adjust(), ad_omega_mod::ad_omega(), ad_ini_fields_mod::ad_out_fields(), ad_ini_fields_mod::ad_out_zeta(), ad_output(), ad_pack(), ad_pack_tile(), ad_pre_step3d_mod::ad_pre_step3d(), ad_prsgrd_mod::ad_prsgrd(), ad_rho_eos_mod::ad_rho_eos(), ad_rhs3d_mod::ad_rhs3d(), ad_set_avg_mod::ad_set_avg(), ad_set_data(), ad_set_depth_mod::ad_set_depth(), ad_set_depth_mod::ad_set_depth_bry(), ad_set_massflux_mod::ad_set_massflux(), ad_set_vbc_mod::ad_set_vbc(), ad_set_vbc_mod::ad_set_vbc_tile(), ad_set_zeta_mod::ad_set_zeta(), ad_ini_fields_mod::ad_set_zeta_timeavg(), dotproduct_mod::ad_statenorm(), ad_step2d_mod::ad_step2d(), ad_step3d_t_mod::ad_step3d_t(), ad_step3d_uv_mod::ad_step3d_uv(), ad_t3dmix2_mod::ad_t3dmix2(), ad_t3dmix4_mod::ad_t3dmix4(), ad_t3drelax_mod::ad_t3drelax(), ad_unpack(), ad_unpack_tile(), ad_uv3dmix2_mod::ad_uv3dmix2(), ad_uv3dmix4_mod::ad_uv3dmix4(), ad_uv3drelax_mod::ad_uv3drelax(), uv_var_change_mod::ad_uv_a2c_grid(), uv_var_change_mod::ad_uv_c2a_grid(), roms_kernel_mod::adm_initial(), i4dvar_mod::analysis(), r4dvar_mod::analysis(), rbl4dvar_mod::analysis(), i4dvar_mod::background(), r4dvar_mod::background(), rbl4dvar_mod::background(), rbl4dvar_mod::background_initialize(), bbl_mod::bblm(), biology_mod::biology(), biology_floats_mod::biology_floats(), bulk_flux_mod::bulk_flux(), cgradient_mod::cgradient(), congrad_mod::congrad(), convolve_mod::convolve(), diag_mod::diag(), convolve_mod::error_covariance(), frc_weak_mod::frc_adgather(), frc_adjust_mod::frc_adjust(), frc_weak_mod::frc_clear(), frc_iau_mod::frc_iau(), frc_iau_mod::frc_iau_ini(), get_data(), get_idata(), get_state_mod::get_state_nf90(), get_state_mod::get_state_pio(), gls_corstep_mod::gls_corstep(), gls_prestep_mod::gls_prestep(), ice_advect_mod::ice_advect(), ice_thermo_mod::ice_thermo(), i4dvar_mod::increment(), r4dvar_mod::increment(), rbl4dvar_mod::increment(), ini_adjust_mod::ini_adjust(), ini_adjust_mod::ini_adjust_tile(), inner2state_mod::ini_c_norm(), ini_fields_mod::ini_fields(), ini_lanczos_mod::ini_lanczos(), ini_adjust_mod::ini_perturb(), ini_fields_mod::ini_zeta(), initial(), lmd_vmix_mod::lmd_vmix(), ini_adjust_mod::load_adtotl(), frc_adjust_mod::load_frc(), obc_adjust_mod::load_obc(), ini_adjust_mod::load_tltoad(), distribute_mod::mp_aggregate2d(), distribute_mod::mp_aggregate3d(), distribute_mod::mp_assemble::mp_assemblef_1d(), distribute_mod::mp_assemble::mp_assemblef_2d(), distribute_mod::mp_assemble::mp_assemblef_3d(), distribute_mod::mp_assemble::mp_assemblei_1d(), distribute_mod::mp_assemble::mp_assemblei_2d(), distribute_mod::mp_barrier(), distribute_mod::mp_bcast_struc(), distribute_mod::mp_bcastf::mp_bcastf_0d(), distribute_mod::mp_bcastf::mp_bcastf_0dp(), distribute_mod::mp_bcastf::mp_bcastf_1d(), distribute_mod::mp_bcastf::mp_bcastf_1dp(), distribute_mod::mp_bcastf::mp_bcastf_2d(), distribute_mod::mp_bcastf::mp_bcastf_2dp(), distribute_mod::mp_bcastf::mp_bcastf_3d(), distribute_mod::mp_bcastf::mp_bcastf_3dp(), distribute_mod::mp_bcastf::mp_bcastf_4d(), distribute_mod::mp_bcasti::mp_bcasti_0d(), distribute_mod::mp_bcasti::mp_bcasti_1d(), distribute_mod::mp_bcasti::mp_bcasti_2d(), distribute_mod::mp_bcastl::mp_bcastl_0d(), distribute_mod::mp_bcastl::mp_bcastl_1d(), distribute_mod::mp_bcastl::mp_bcastl_2d(), distribute_mod::mp_bcasts::mp_bcasts_0d(), distribute_mod::mp_bcasts::mp_bcasts_1d(), distribute_mod::mp_bcasts::mp_bcasts_2d(), distribute_mod::mp_bcasts::mp_bcasts_3d(), distribute_mod::mp_boundary(), distribute_mod::mp_collect::mp_collect_f(), distribute_mod::mp_collect::mp_collect_i(), mp_exchange_mod::mp_exchange2d(), mp_exchange_mod::mp_exchange2d_bry(), mp_exchange_mod::mp_exchange3d(), mp_exchange_mod::mp_exchange3d_bry(), mp_exchange_mod::mp_exchange4d(), distribute_mod::mp_gather2d(), distribute_mod::mp_gather3d(), distribute_mod::mp_gather_state(), distribute_mod::mp_ncread1d(), distribute_mod::mp_ncread2d(), distribute_mod::mp_ncwrite1d(), distribute_mod::mp_ncwrite2d(), distribute_mod::mp_reduce2(), distribute_mod::mp_reduce::mp_reduce_0d(), distribute_mod::mp_reduce::mp_reduce_0dp(), distribute_mod::mp_reduce::mp_reduce_1d(), distribute_mod::mp_reduce::mp_reduce_1dp(), distribute_mod::mp_reduce::mp_reduce_i8(), distribute_mod::mp_scatter2d(), distribute_mod::mp_scatter3d(), distribute_mod::mp_scatter_state(), my25_corstep_mod::my25_corstep(), my25_prestep_mod::my25_prestep(), nesting_mod::nesting(), dotproduct_mod::nl_dotproduct(), roms_kernel_mod::nlm_initial(), obc_adjust_mod::obc_adjust(), omega_mod::omega(), output(), posterior_mod::posterior(), i4dvar_mod::posterior_analysis(), i4dvar_mod::posterior_analysis_initialize(), posterior_var_mod::posterior_var(), pre_step3d_mod::pre_step3d(), prsgrd_mod::prsgrd(), random_ic_mod::random_ic(), set_massflux_mod::reset_massflux(), rho_eos_mod::rho_eos(), rhs3d_mod::rhs3d(), mod_arrays::roms_allocate_arrays(), roms_kernel_mod::roms_finalize(), mod_arrays::roms_initialize_arrays(), roms_kernel_mod::roms_run(), roms_kernel_mod::roms_run(), rp_set_depth_mod::rp_bath(), rp_biology_mod::rp_biology(), rp_bulk_flux_mod::rp_bulk_flux(), rp_diag_mod::rp_diag(), rp_frc_adjust_mod::rp_frc_adjust(), rp_get_data(), rp_get_idata(), ini_adjust_mod::rp_ini_adjust(), rp_ini_fields_mod::rp_ini_fields(), rp_ini_fields_mod::rp_ini_zeta(), rp_initial(), rp_obc_adjust_mod::rp_obc2d_adjust(), rp_obc_adjust_mod::rp_obc_adjust(), rp_omega_mod::rp_omega(), rp_output(), rp_pre_step3d_mod::rp_pre_step3d(), rp_prsgrd_mod::rp_prsgrd(), rp_rho_eos_mod::rp_rho_eos(), rp_rhs3d_mod::rp_rhs3d(), rp_set_data(), rp_set_depth_mod::rp_set_depth(), rp_set_depth_mod::rp_set_depth_bry(), rp_set_massflux_mod::rp_set_massflux(), rp_set_vbc_mod::rp_set_vbc(), rp_set_vbc_mod::rp_set_vbc_tile(), rp_set_zeta_mod::rp_set_zeta(), rp_ini_fields_mod::rp_set_zeta_timeavg(), rp_step2d_mod::rp_step2d(), rp_step3d_t_mod::rp_step3d_t(), rp_step3d_uv_mod::rp_step3d_uv(), rp_t3dmix2_mod::rp_t3dmix2(), rp_t3dmix4_mod::rp_t3dmix4(), rp_t3drelax_mod::rp_t3drelax(), rp_uv3dmix2_mod::rp_uv3dmix2(), rp_uv3dmix4_mod::rp_uv3dmix4(), rp_uv3drelax_mod::rp_uv3drelax(), rpcg_lanczos_mod::rpcg_lanczos(), convolve_mod::saddlec(), sed_bed_mod::sed_bed(), sed_bedload(), sed_fluxes_mod::sed_fluxes(), sed_settling_mod::sed_settling(), sed_surface_mod::sed_surface(), set_avg_mod::set_avg(), set_data(), set_depth_mod::set_depth(), set_depth_mod::set_depth0(), set_depth_mod::set_depth_bry(), set_diags(), set_masks_mod::set_masks(), set_massflux_mod::set_massflux(), set_tides_mod::set_tides(), set_vbc_mod::set_vbc(), set_vbc_mod::set_vbc_tile(), set_zeta_mod::set_zeta(), ini_fields_mod::set_zeta_timeavg(), state_read_mod::state_read_nf90(), state_read_mod::state_read_pio(), step2d_mod::step2d(), step3d_t_mod::step3d_t(), step3d_uv_mod::step3d_uv(), step_floats_mod::step_floats(), t3dmix2_mod::t3dmix2(), t3dmix4_mod::t3dmix4(), tl_set_depth_mod::tl_bath(), tl_biology_mod::tl_biology(), tl_bulk_flux_mod::tl_bulk_flux(), tl_diag_mod::tl_diag(), dotproduct_mod::tl_dotproduct(), tl_frc_adjust_mod::tl_frc_adjust(), tl_get_data(), tl_get_idata(), tl_ini_fields_mod::tl_ini_fields(), ini_adjust_mod::tl_ini_perturb(), tl_ini_fields_mod::tl_ini_zeta(), tl_initial(), inner2state_mod::tl_inner2state(), tl_nesting_mod::tl_nesting(), tl_obc_adjust_mod::tl_obc2d_adjust(), tl_obc_adjust_mod::tl_obc_adjust(), tl_omega_mod::tl_omega(), tl_output(), tl_pack(), tl_pre_step3d_mod::tl_pre_step3d(), tl_prsgrd_mod::tl_prsgrd(), tl_rho_eos_mod::tl_rho_eos(), tl_rhs3d_mod::tl_rhs3d(), tl_set_avg_mod::tl_set_avg(), tl_set_data(), tl_set_depth_mod::tl_set_depth(), tl_set_depth_mod::tl_set_depth_bry(), tl_set_massflux_mod::tl_set_massflux(), tl_set_vbc_mod::tl_set_vbc(), tl_set_vbc_mod::tl_set_vbc_tile(), tl_set_zeta_mod::tl_set_zeta(), tl_ini_fields_mod::tl_set_zeta_timeavg(), dotproduct_mod::tl_statenorm(), tl_step2d_mod::tl_step2d(), tl_step3d_t_mod::tl_step3d_t(), tl_step3d_uv_mod::tl_step3d_uv(), tl_t3dmix2_mod::tl_t3dmix2(), tl_t3dmix4_mod::tl_t3dmix4(), tl_t3drelax_mod::tl_t3drelax(), tl_unpack(), tl_unpack_tile(), tl_uv3dmix2_mod::tl_uv3dmix2(), tl_uv3dmix4_mod::tl_uv3dmix4(), tl_uv3drelax_mod::tl_uv3drelax(), uv_var_change_mod::tl_uv_a2c_grid(), uv_var_change_mod::tl_uv_c2a_grid(), roms_kernel_mod::tlm_initial(), uv3dmix2_mod::uv3dmix2(), uv3dmix4_mod::uv3dmix4(), uv_var_change_mod::uv_a2c_grid(), uv_var_change_mod::uv_c2a_grid(), vorticity_mod::vorticity(), vwalk_floats_mod::vwalk_floats(), wrt_state_mod::wrt_state_nf90(), and wrt_state_mod::wrt_state_pio().

Here is the call graph for this function:

Here is the caller graph for this function:

wclock_on()

recursive subroutine wclock_on	(	integer, intent(in)	ng,
		integer, intent(in)	model,
		integer, intent(in)	region,
		integer, intent(in)	line,
		character (len=*), intent(in)	routine )

Definition at line 2 of file timers.F.

!
 
!git $Id$
!================================================== Hernan G. Arango ===
!  Copyright (c) 2002-2025 The ROMS Group                              !
!    Licensed under a MIT/X style license                              !
!    See License_ROMS.md                                               !
!=======================================================================
!                                                                      !
!  This routine turns on wall clock to meassure the elapsed time in    !
!  seconds spend by each parallel thread in requested model region.    !
!                                                                      !
!  On Input:                                                           !
!                                                                      !
!     ng         Nested grid number (integer)                          !
!     model      Calling model identifier (integer)                    !
!     region     Profiling reagion number (integer)                    !
!     line       Calling model routine line (integer)                  !
!     routine    Calling model routine (string)                        !
!                                                                      !
!=======================================================================
!
      USE mod_param
      USE mod_parallel
      USE mod_iounits
      USE mod_strings
 
#ifdef DISTRIBUTE
!
      USE distribute_mod, ONLY : mp_barrier
# ifdef ROMS_STDOUT
      USE distribute_mod, ONLY : mp_collect
# endif
#endif
!
      implicit none
!
!  Imported variable declarations.
!
      integer, intent(in) :: ng, model, region, line
 
      character (len=*), intent(in) :: routine
!
!  Local variable declarations.
!
      integer :: iregion, MyModel, NSUB
 
      integer :: my_getpid
#ifndef DISTRIBUTE
      integer :: my_threadnum
#else
      integer :: MyCOMM, nPETs, PETrank
# ifdef ROMS_STDOUT
      integer :: node
      integer, parameter :: Pspv = 0
      integer, allocatable :: Pids(:)
# endif
#endif
      real(r8), dimension(2) :: wtime
      real(r8) :: my_wtime
!
!-----------------------------------------------------------------------
!  Initialize timing for all threads.
!-----------------------------------------------------------------------
#ifdef TRACING
!
      IF (master) THEN
        WRITE (stdout,'(a,a)') '==> Entering ', trim(routine)
        FLUSH (stdout)
      END IF
#endif
!
!  Set number of subdivisions, same as for global reductions.
!
#ifdef DISTRIBUTE
# ifdef DISJOINTED
      mycomm=full_comm_world
      npets=fullsize
      petrank=fullrank
# else
      mycomm=ocn_comm_world
      npets=numthreads
      petrank=myrank
# endif
      nsub=1
#else
      nsub=numthreads
#endif
!
!  Insure that MyModel is not zero.
!
      mymodel=max(1,model)
!
!  Start the wall CPU clock for specified region, model, and grid.
!
      cstr(region,mymodel,ng)=my_wtime(wtime)
!
!  If region zero, indicating first call from main driver, initialize
!  time profiling arrays and set process ID.
!
      IF ((region.eq.0).and.(proc(1,mymodel,ng).eq.0)) THEN
        DO iregion=1,nregion
          cend(iregion,mymodel,ng)=0.0_r8
          csum(iregion,mymodel,ng)=0.0_r8
        END DO
        proc(1,mymodel,ng)=1
        proc(0,mymodel,ng)=my_getpid()
 
!$OMP CRITICAL (START_WCLOCK)
        IF (ng.eq.1) THEN
#ifdef DISTRIBUTE
          CALL mp_barrier (ng, model, mycomm)
# ifdef ROMS_STDOUT
          IF (.not.allocated(pids)) THEN
            allocate ( pids(npets) )
            pids=0
          END IF
          pids(petrank+1)=proc(0,mymodel,ng)
          CALL mp_collect (ng, model, npets, pspv, pids, mycomm)
          IF (master) THEN
            DO node=1,npets
              WRITE (stdout,10) ' Node #', node-1,                      &
     &                      ' (pid=',pids(node),') is active.'
            END DO
          END IF
          IF (allocated(pids)) deallocate (pids)
# else
          WRITE (stdout,10) ' Node #', petrank,                         &
     &                      ' (pid=',proc(0,mymodel,ng),') is active.'
# endif
          FLUSH (stdout)
#else
          WRITE (stdout,10) ' Thread #', mythread,                      &
     &                      ' (pid=',proc(0,mymodel,ng),') is active.'
#endif
        END IF
 10     FORMAT (a,i5,a,i8,a)
        thread_count=thread_count+1
        IF (thread_count.eq.nsub) thread_count=0
!$OMP END CRITICAL (START_WCLOCK)
      END IF
      RETURN

References mod_parallel::cend, mod_parallel::cstr, mod_parallel::csum, mod_parallel::fullrank, mod_parallel::master, distribute_mod::mp_barrier(), my_getpid(), my_wtime(), mod_parallel::myrank, mod_parallel::mythread, mod_strings::nregion, mod_parallel::numthreads, mod_parallel::ocn_comm_world, mod_parallel::proc, mod_iounits::stdout, and mod_parallel::thread_count.

Referenced by ad_biology_mod::ad_biology(), ad_bulk_flux_mod::ad_bulk_flux(), ad_diag_mod::ad_diag(), dotproduct_mod::ad_dotproduct(), ad_frc_adjust_mod::ad_frc_adjust(), ad_get_data(), ad_get_idata(), ad_ini_fields_mod::ad_ini_fields(), ini_adjust_mod::ad_ini_perturb(), ad_ini_fields_mod::ad_ini_zeta(), ad_initial(), inner2state_mod::ad_inner2state(), mp_exchange_mod::ad_mp_exchange2d(), mp_exchange_mod::ad_mp_exchange2d_bry(), mp_exchange_mod::ad_mp_exchange3d(), mp_exchange_mod::ad_mp_exchange3d_bry(), mp_exchange_mod::ad_mp_exchange4d(), ad_obc_adjust_mod::ad_obc2d_adjust(), ad_obc_adjust_mod::ad_obc_adjust(), ad_omega_mod::ad_omega(), ad_ini_fields_mod::ad_out_fields(), ad_ini_fields_mod::ad_out_zeta(), ad_output(), ad_pack(), ad_pack_tile(), ad_pre_step3d_mod::ad_pre_step3d(), ad_prsgrd_mod::ad_prsgrd(), ad_rho_eos_mod::ad_rho_eos(), ad_rhs3d_mod::ad_rhs3d(), ad_set_avg_mod::ad_set_avg(), ad_set_data(), ad_set_depth_mod::ad_set_depth(), ad_set_depth_mod::ad_set_depth_bry(), ad_set_massflux_mod::ad_set_massflux(), ad_set_vbc_mod::ad_set_vbc(), ad_set_vbc_mod::ad_set_vbc_tile(), ad_set_zeta_mod::ad_set_zeta(), ad_ini_fields_mod::ad_set_zeta_timeavg(), dotproduct_mod::ad_statenorm(), ad_step2d_mod::ad_step2d(), ad_step3d_t_mod::ad_step3d_t(), ad_step3d_uv_mod::ad_step3d_uv(), ad_t3dmix2_mod::ad_t3dmix2(), ad_t3dmix4_mod::ad_t3dmix4(), ad_t3drelax_mod::ad_t3drelax(), ad_unpack(), ad_unpack_tile(), ad_uv3dmix2_mod::ad_uv3dmix2(), ad_uv3dmix4_mod::ad_uv3dmix4(), ad_uv3drelax_mod::ad_uv3drelax(), uv_var_change_mod::ad_uv_a2c_grid(), uv_var_change_mod::ad_uv_c2a_grid(), roms_kernel_mod::adm_initial(), i4dvar_mod::analysis(), r4dvar_mod::analysis(), rbl4dvar_mod::analysis(), rbl4dvar_mod::analysis_initialize(), i4dvar_mod::background(), r4dvar_mod::background(), rbl4dvar_mod::background(), i4dvar_mod::background_initialize(), rbl4dvar_mod::background_initialize(), bbl_mod::bblm(), biology_mod::biology(), biology_floats_mod::biology_floats(), bulk_flux_mod::bulk_flux(), cgradient_mod::cgradient(), congrad_mod::congrad(), convolve_mod::convolve(), diag_mod::diag(), convolve_mod::error_covariance(), frc_weak_mod::frc_adgather(), frc_adjust_mod::frc_adjust(), frc_weak_mod::frc_clear(), frc_iau_mod::frc_iau(), frc_iau_mod::frc_iau_ini(), get_data(), get_idata(), get_state_mod::get_state_nf90(), get_state_mod::get_state_pio(), gls_corstep_mod::gls_corstep(), gls_prestep_mod::gls_prestep(), ice_advect_mod::ice_advect(), ice_thermo_mod::ice_thermo(), i4dvar_mod::increment(), r4dvar_mod::increment(), rbl4dvar_mod::increment(), ini_adjust_mod::ini_adjust(), ini_adjust_mod::ini_adjust_tile(), inner2state_mod::ini_c_norm(), ini_fields_mod::ini_fields(), ini_lanczos_mod::ini_lanczos(), ini_adjust_mod::ini_perturb(), ini_fields_mod::ini_zeta(), initial(), lmd_vmix_mod::lmd_vmix(), ini_adjust_mod::load_adtotl(), frc_adjust_mod::load_frc(), obc_adjust_mod::load_obc(), ini_adjust_mod::load_tltoad(), distribute_mod::mp_aggregate2d(), distribute_mod::mp_aggregate3d(), distribute_mod::mp_assemble::mp_assemblef_1d(), distribute_mod::mp_assemble::mp_assemblef_2d(), distribute_mod::mp_assemble::mp_assemblef_3d(), distribute_mod::mp_assemble::mp_assemblei_1d(), distribute_mod::mp_assemble::mp_assemblei_2d(), distribute_mod::mp_barrier(), distribute_mod::mp_bcast_struc(), distribute_mod::mp_bcastf::mp_bcastf_0d(), distribute_mod::mp_bcastf::mp_bcastf_0dp(), distribute_mod::mp_bcastf::mp_bcastf_1d(), distribute_mod::mp_bcastf::mp_bcastf_1dp(), distribute_mod::mp_bcastf::mp_bcastf_2d(), distribute_mod::mp_bcastf::mp_bcastf_2dp(), distribute_mod::mp_bcastf::mp_bcastf_3d(), distribute_mod::mp_bcastf::mp_bcastf_3dp(), distribute_mod::mp_bcastf::mp_bcastf_4d(), distribute_mod::mp_bcasti::mp_bcasti_0d(), distribute_mod::mp_bcasti::mp_bcasti_1d(), distribute_mod::mp_bcasti::mp_bcasti_2d(), distribute_mod::mp_bcastl::mp_bcastl_0d(), distribute_mod::mp_bcastl::mp_bcastl_1d(), distribute_mod::mp_bcastl::mp_bcastl_2d(), distribute_mod::mp_bcasts::mp_bcasts_0d(), distribute_mod::mp_bcasts::mp_bcasts_1d(), distribute_mod::mp_bcasts::mp_bcasts_2d(), distribute_mod::mp_bcasts::mp_bcasts_3d(), distribute_mod::mp_boundary(), distribute_mod::mp_collect::mp_collect_f(), distribute_mod::mp_collect::mp_collect_i(), mp_exchange_mod::mp_exchange2d(), mp_exchange_mod::mp_exchange2d_bry(), mp_exchange_mod::mp_exchange3d(), mp_exchange_mod::mp_exchange3d_bry(), mp_exchange_mod::mp_exchange4d(), distribute_mod::mp_gather2d(), distribute_mod::mp_gather3d(), distribute_mod::mp_gather_state(), distribute_mod::mp_ncread1d(), distribute_mod::mp_ncread2d(), distribute_mod::mp_ncwrite1d(), distribute_mod::mp_ncwrite2d(), distribute_mod::mp_reduce2(), distribute_mod::mp_reduce::mp_reduce_0d(), distribute_mod::mp_reduce::mp_reduce_0dp(), distribute_mod::mp_reduce::mp_reduce_1d(), distribute_mod::mp_reduce::mp_reduce_1dp(), distribute_mod::mp_reduce::mp_reduce_i8(), distribute_mod::mp_scatter2d(), distribute_mod::mp_scatter3d(), distribute_mod::mp_scatter_state(), my25_corstep_mod::my25_corstep(), my25_prestep_mod::my25_prestep(), nesting_mod::nesting(), dotproduct_mod::nl_dotproduct(), roms_kernel_mod::nlm_initial(), obc_adjust_mod::obc_adjust(), omega_mod::omega(), output(), posterior_mod::posterior(), i4dvar_mod::posterior_analysis(), i4dvar_mod::posterior_analysis_initialize(), posterior_var_mod::posterior_var(), pre_step3d_mod::pre_step3d(), prsgrd_mod::prsgrd(), random_ic_mod::random_ic(), set_massflux_mod::reset_massflux(), rho_eos_mod::rho_eos(), rhs3d_mod::rhs3d(), mod_arrays::roms_allocate_arrays(), roms_kernel_mod::roms_initialize(), mod_arrays::roms_initialize_arrays(), roms_kernel_mod::roms_initializep1(), rp_set_depth_mod::rp_bath(), rp_biology_mod::rp_biology(), rp_bulk_flux_mod::rp_bulk_flux(), rp_diag_mod::rp_diag(), rp_frc_adjust_mod::rp_frc_adjust(), rp_get_data(), rp_get_idata(), ini_adjust_mod::rp_ini_adjust(), rp_ini_fields_mod::rp_ini_fields(), rp_ini_fields_mod::rp_ini_zeta(), rp_initial(), rp_obc_adjust_mod::rp_obc2d_adjust(), rp_obc_adjust_mod::rp_obc_adjust(), rp_omega_mod::rp_omega(), rp_output(), rp_pre_step3d_mod::rp_pre_step3d(), rp_prsgrd_mod::rp_prsgrd(), rp_rho_eos_mod::rp_rho_eos(), rp_rhs3d_mod::rp_rhs3d(), rp_set_data(), rp_set_depth_mod::rp_set_depth(), rp_set_depth_mod::rp_set_depth_bry(), rp_set_massflux_mod::rp_set_massflux(), rp_set_vbc_mod::rp_set_vbc(), rp_set_vbc_mod::rp_set_vbc_tile(), rp_set_zeta_mod::rp_set_zeta(), rp_ini_fields_mod::rp_set_zeta_timeavg(), rp_step2d_mod::rp_step2d(), rp_step3d_t_mod::rp_step3d_t(), rp_step3d_uv_mod::rp_step3d_uv(), rp_t3dmix2_mod::rp_t3dmix2(), rp_t3dmix4_mod::rp_t3dmix4(), rp_t3drelax_mod::rp_t3drelax(), rp_uv3dmix2_mod::rp_uv3dmix2(), rp_uv3dmix4_mod::rp_uv3dmix4(), rp_uv3drelax_mod::rp_uv3drelax(), rpcg_lanczos_mod::rpcg_lanczos(), convolve_mod::saddlec(), sed_bed_mod::sed_bed(), sed_bedload(), sed_fluxes_mod::sed_fluxes(), sed_settling_mod::sed_settling(), sed_surface_mod::sed_surface(), set_avg_mod::set_avg(), set_data(), set_depth_mod::set_depth(), set_depth_mod::set_depth0(), set_depth_mod::set_depth_bry(), set_diags(), set_masks_mod::set_masks(), set_massflux_mod::set_massflux(), set_tides_mod::set_tides(), set_vbc_mod::set_vbc(), set_vbc_mod::set_vbc_tile(), set_zeta_mod::set_zeta(), ini_fields_mod::set_zeta_timeavg(), state_read_mod::state_read_nf90(), state_read_mod::state_read_pio(), step2d_mod::step2d(), step3d_t_mod::step3d_t(), step3d_uv_mod::step3d_uv(), step_floats_mod::step_floats(), t3dmix2_mod::t3dmix2(), t3dmix4_mod::t3dmix4(), tl_set_depth_mod::tl_bath(), tl_biology_mod::tl_biology(), tl_bulk_flux_mod::tl_bulk_flux(), tl_diag_mod::tl_diag(), dotproduct_mod::tl_dotproduct(), tl_frc_adjust_mod::tl_frc_adjust(), tl_get_data(), tl_get_idata(), tl_ini_fields_mod::tl_ini_fields(), ini_adjust_mod::tl_ini_perturb(), tl_ini_fields_mod::tl_ini_zeta(), tl_initial(), inner2state_mod::tl_inner2state(), tl_nesting_mod::tl_nesting(), tl_obc_adjust_mod::tl_obc2d_adjust(), tl_obc_adjust_mod::tl_obc_adjust(), tl_omega_mod::tl_omega(), tl_output(), tl_pack(), tl_pre_step3d_mod::tl_pre_step3d(), tl_prsgrd_mod::tl_prsgrd(), tl_rho_eos_mod::tl_rho_eos(), tl_rhs3d_mod::tl_rhs3d(), tl_set_avg_mod::tl_set_avg(), tl_set_data(), tl_set_depth_mod::tl_set_depth(), tl_set_depth_mod::tl_set_depth_bry(), tl_set_massflux_mod::tl_set_massflux(), tl_set_vbc_mod::tl_set_vbc(), tl_set_vbc_mod::tl_set_vbc_tile(), tl_set_zeta_mod::tl_set_zeta(), tl_ini_fields_mod::tl_set_zeta_timeavg(), dotproduct_mod::tl_statenorm(), tl_step2d_mod::tl_step2d(), tl_step3d_t_mod::tl_step3d_t(), tl_step3d_uv_mod::tl_step3d_uv(), tl_t3dmix2_mod::tl_t3dmix2(), tl_t3dmix4_mod::tl_t3dmix4(), tl_t3drelax_mod::tl_t3drelax(), tl_unpack(), tl_unpack_tile(), tl_uv3dmix2_mod::tl_uv3dmix2(), tl_uv3dmix4_mod::tl_uv3dmix4(), tl_uv3drelax_mod::tl_uv3drelax(), uv_var_change_mod::tl_uv_a2c_grid(), uv_var_change_mod::tl_uv_c2a_grid(), roms_kernel_mod::tlm_initial(), uv3dmix2_mod::uv3dmix2(), uv3dmix4_mod::uv3dmix4(), uv_var_change_mod::uv_a2c_grid(), uv_var_change_mod::uv_c2a_grid(), vorticity_mod::vorticity(), vwalk_floats_mod::vwalk_floats(), wrt_state_mod::wrt_state_nf90(), and wrt_state_mod::wrt_state_pio().

Here is the call graph for this function:

Here is the caller graph for this function: