rdb_halo Module

Exchanges ghost-cell strips between neighbouring MPI ranks

Uses pic_mpi_lib for non-blocking sends/receives. Plus-shaped stencil: 4 exchanges (N/S/E/W), no corner exchanges.

Three exchange modes: halo_exchange_2d — host-staged, blocking (caller copies device<->host) halo_exchange_2d_device — GPU-direct, blocking (CUDA-aware MPI) halo_exchange_begin/end — async split for comm/compute overlap


Uses

  • module~~rdb_halo~~UsesGraph module~rdb_halo rdb_halo iso_fortran_env iso_fortran_env module~rdb_halo->iso_fortran_env module~rdb_comm_env rdb_comm_env module~rdb_halo->module~rdb_comm_env module~rdb_constants rdb_constants module~rdb_halo->module~rdb_constants module~rdb_decomp rdb_decomp module~rdb_halo->module~rdb_decomp module~rdb_efp rdb_efp module~rdb_halo->module~rdb_efp pic_logger pic_logger module~rdb_halo->pic_logger pic_mpi_lib pic_mpi_lib module~rdb_halo->pic_mpi_lib module~rdb_comm_env->iso_fortran_env module~rdb_comm_env->module~rdb_constants module~rdb_comm_env->pic_mpi_lib pic_types pic_types module~rdb_constants->pic_types module~rdb_config rdb_config module~rdb_decomp->module~rdb_config module~rdb_efp->iso_fortran_env ieee_arithmetic ieee_arithmetic module~rdb_efp->ieee_arithmetic module~rdb_config->module~rdb_constants module~rdb_config->pic_logger module~rdb_error_ring rdb_error_ring module~rdb_config->module~rdb_error_ring module~rdb_ice_enthalpy rdb_ice_enthalpy module~rdb_config->module~rdb_ice_enthalpy module~rdb_ice_init rdb_ice_init module~rdb_config->module~rdb_ice_init module~rdb_nml_schema rdb_nml_schema module~rdb_config->module~rdb_nml_schema module~rdb_ocean_status rdb_ocean_status module~rdb_config->module~rdb_ocean_status pic_ascii pic_ascii module~rdb_config->pic_ascii pic_strings pic_strings module~rdb_config->pic_strings module~rdb_error_ring->pic_logger module~rdb_ice_enthalpy->module~rdb_constants module~rdb_ice_init->module~rdb_constants module~rdb_ice_init->module~rdb_ice_enthalpy module~rdb_grid rdb_grid module~rdb_ice_init->module~rdb_grid module~rdb_ice_column rdb_ice_column module~rdb_ice_init->module~rdb_ice_column module~rdb_ice_state rdb_ice_state module~rdb_ice_init->module~rdb_ice_state module~rdb_multilayer_state rdb_multilayer_state module~rdb_ice_init->module~rdb_multilayer_state module~rdb_ocean_metrics rdb_ocean_metrics module~rdb_ice_init->module~rdb_ocean_metrics module~rdb_nml_schema->module~rdb_constants module~rdb_nml_schema->pic_logger module~rdb_nml_schema->module~rdb_error_ring module~rdb_grid->module~rdb_constants module~rdb_ice_column->module~rdb_constants module~rdb_ice_column->module~rdb_ice_enthalpy module~rdb_ice_mass rdb_ice_mass module~rdb_ice_column->module~rdb_ice_mass module~rdb_ice_optics rdb_ice_optics module~rdb_ice_column->module~rdb_ice_optics module~rdb_ice_state->iso_fortran_env module~rdb_ice_state->module~rdb_constants module~rdb_ice_state->module~rdb_ice_enthalpy module~rdb_ice_state->module~rdb_grid module~rdb_ice_state->module~rdb_ice_column module~rdb_mem_report rdb_mem_report module~rdb_ice_state->module~rdb_mem_report module~rdb_multilayer_state->iso_fortran_env module~rdb_multilayer_state->module~rdb_constants module~rdb_multilayer_state->module~rdb_efp module~rdb_multilayer_state->pic_logger module~rdb_multilayer_state->module~rdb_error_ring module~rdb_multilayer_state->module~rdb_grid module~rdb_multilayer_state->module~rdb_mem_report module~rdb_tracer rdb_tracer module~rdb_multilayer_state->module~rdb_tracer module~rdb_ocean_metrics->iso_fortran_env module~rdb_ocean_metrics->module~rdb_constants module~rdb_ocean_metrics->pic_logger module~rdb_ocean_metrics->module~rdb_error_ring module~rdb_ocean_metrics->module~rdb_ocean_status module~rdb_ocean_metrics->pic_strings module~rdb_ocean_metrics->module~rdb_grid module~rdb_io_netcdf rdb_io_netcdf module~rdb_ocean_metrics->module~rdb_io_netcdf module~rdb_ocean_metrics->module~rdb_mem_report module~rdb_ocean_bipolar rdb_ocean_bipolar module~rdb_ocean_metrics->module~rdb_ocean_bipolar module~rdb_ocean_fold rdb_ocean_fold module~rdb_ocean_metrics->module~rdb_ocean_fold netcdf netcdf module~rdb_ocean_metrics->netcdf

Used by

  • module~~rdb_halo~~UsedByGraph module~rdb_halo rdb_halo module~rdb_driver rdb_driver module~rdb_driver->module~rdb_halo module~rdb_ocean_console_stats rdb_ocean_console_stats module~rdb_driver->module~rdb_ocean_console_stats module~rdb_ocean_engine rdb_ocean_engine module~rdb_driver->module~rdb_ocean_engine module~rdb_ocean_dyn rdb_ocean_dyn module~rdb_driver->module~rdb_ocean_dyn module~rdb_ocean_state rdb_ocean_state module~rdb_driver->module~rdb_ocean_state module~rdb_ice_transport rdb_ice_transport module~rdb_ice_transport->module~rdb_halo module~rdb_ocean_chksum rdb_ocean_chksum module~rdb_ocean_chksum->module~rdb_halo module~rdb_ocean_console_stats->module~rdb_halo module~rdb_ocean_engine->module~rdb_halo module~rdb_ocean_engine->module~rdb_ice_transport module~rdb_ocean_setup rdb_ocean_setup module~rdb_ocean_engine->module~rdb_ocean_setup module~rdb_ocean_engine->module~rdb_ocean_dyn module~rdb_ocean_engine->module~rdb_ocean_state module~rdb_ocean_diag_derived rdb_ocean_diag_derived module~rdb_ocean_engine->module~rdb_ocean_diag_derived module~rdb_ocean_diag_fills rdb_ocean_diag_fills module~rdb_ocean_engine->module~rdb_ocean_diag_fills module~rdb_ocean_setup->module~rdb_halo module~rdb_ocean_setup->module~rdb_ocean_dyn module~rdb_ocean_setup->module~rdb_ocean_state proc~ocean_cavity_mass_step ocean_cavity_mass_step proc~ocean_cavity_mass_step->module~rdb_halo module~rdb_handle rdb_handle module~rdb_handle->module~rdb_ocean_engine module~rdb_handle->module~rdb_ocean_state module~rdb_ocean_api rdb_ocean_api module~rdb_ocean_api->module~rdb_ocean_engine module~rdb_ocean_api->module~rdb_handle module~rdb_ocean_api->module~rdb_ocean_dyn module~rdb_ocean_api->module~rdb_ocean_diag_derived module~rdb_ocean_api->module~rdb_ocean_diag_fills module~rdb_ocean_dyn->module~rdb_ocean_chksum module~rdb_ocean_dyn->module~rdb_ocean_console_stats module~rdb_ocean_state->module~rdb_ocean_dyn module~rdb_ocean_diag_derived->module~rdb_ocean_state module~rdb_ocean_diag_derived->module~rdb_ocean_diag_fills module~rdb_ocean_diag_fills->module~rdb_ocean_state

Variables

Type Visibility Attributes Name Initial
integer, private, parameter :: MAX_REQS = 8

Max MPI requests: 4 directions x 2 (send + recv)
integer, private, parameter :: NFIELDS = 4

Number of fields exchanged (h, hu, hv, b)
real(kind=wp), private, allocatable :: ha_buf_recv_east(:)
real(kind=wp), private, allocatable :: ha_buf_recv_north(:)
real(kind=wp), private, allocatable :: ha_buf_recv_south(:)
real(kind=wp), private, allocatable :: ha_buf_recv_west(:)
real(kind=wp), private, allocatable :: ha_buf_send_east(:)
real(kind=wp), private, allocatable :: ha_buf_send_north(:)
real(kind=wp), private, allocatable :: ha_buf_send_south(:)
real(kind=wp), private, allocatable :: ha_buf_send_west(:)
real(kind=wp), private, allocatable :: hs3_buf_recv_east(:)
real(kind=wp), private, allocatable :: hs3_buf_recv_north(:)
real(kind=wp), private, allocatable :: hs3_buf_recv_south(:)
real(kind=wp), private, allocatable :: hs3_buf_recv_west(:)
real(kind=wp), private, allocatable :: hs3_buf_send_east(:)
real(kind=wp), private, allocatable :: hs3_buf_send_north(:)
real(kind=wp), private, allocatable :: hs3_buf_send_south(:)
real(kind=wp), private, allocatable :: hs3_buf_send_west(:)
integer, private :: hs3_nghost = 0
integer, private :: hs3_nx_total = 0
integer, private :: hs3_ny_total = 0
integer, private :: hs3_nz_capacity = 0
real(kind=wp), private, allocatable :: hs_buf_recv_east(:)
real(kind=wp), private, allocatable :: hs_buf_recv_north(:)
real(kind=wp), private, allocatable :: hs_buf_recv_south(:)
real(kind=wp), private, allocatable :: hs_buf_recv_west(:)
real(kind=wp), private, allocatable :: hs_buf_send_east(:)
real(kind=wp), private, allocatable :: hs_buf_send_north(:)
real(kind=wp), private, allocatable :: hs_buf_send_south(:)
real(kind=wp), private, allocatable :: hs_buf_send_west(:)
integer, private :: hs_nghost = 0
integer, private :: hs_nx_total = 0
integer, private :: hs_ny_total = 0

Derived Types

type, public ::  halo_async_t

Persistent state for split begin/end halo exchange

Read more…

Components

Type Visibility Attributes Name Initial
type(decomp_t), public :: decomp
logical, public :: initialised = .false.
integer, public :: nghost = 0
integer, public :: nreq = 0
integer, public :: nx_local = 0
integer, public :: nx_total = 0
integer, public :: ny_local = 0
integer, public :: ny_total = 0
integer, public :: rank_east = -1
integer, public :: rank_north = -1
integer, public :: rank_south = -1
integer, public :: rank_west = -1
type(request_t), public :: reqs(MAX_REQS)
integer, public :: strip_ew = 0
integer, public :: strip_sn = 0

Subroutines

public subroutine halo_allreduce_efp_list(local_list, global_list, nval)

Order-invariant EXACT cross-rank combine of nval EFP values in ONE collective (PR-32). Replaces N separate scalar halo_allreduce_sum calls with one packed allreduce.

Read more…

Arguments

Type IntentOptional Attributes Name
type(efp_t), intent(in) :: local_list(:)
type(efp_t), intent(out) :: global_list(:)
integer, intent(in) :: nval

public subroutine halo_allreduce_max(local_val, global_val)

MPI_Allreduce with MPI_MAX — max-type reductions are exact in FP, so a global max stays layout-reproducible (ocean-MPI plan D5). Used for auto_n_inner’s global gravity-wave CFL (shared n_inner).

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(in) :: local_val
real(kind=wp), intent(out) :: global_val

public subroutine halo_allreduce_min(local_val, global_val)

MPI_Allreduce with MPI_MIN for global timestep

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(in) :: local_val
real(kind=wp), intent(out) :: global_val

public subroutine halo_allreduce_sum(local_val, global_val)

MPI_Allreduce with MPI_SUM for CG dot products

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(in) :: local_val
real(kind=wp), intent(out) :: global_val

public subroutine halo_allreduce_sum_i8(local_val, global_val)

Cross-rank int64 sum for the decomposition-invariant chksum bitcount (rdb_ocean_chksum). pic_mpi_lib exposes no integer(int64) allreduce overload (MPI_INTEGER8 reaches only send/recv), so the value rides the EXACT real64 allreduce: a per-field POPCNT sum is bounded by (#elements x 64), which stays FAR below the double-mantissa bound 253 for any realistic grid (253/64 ~ 1.4e14 cells), so both the local->double cast and every partial MPI_SUM are exact — the invariant survives.

Arguments

Type IntentOptional Attributes Name
integer(kind=int64), intent(in) :: local_val
integer(kind=int64), intent(out) :: global_val

public subroutine halo_async_destroy(ha)

Free pre-allocated halo buffers

Arguments

Type IntentOptional Attributes Name
type(halo_async_t), intent(inout) :: ha

public subroutine halo_async_init(ha, decomp, nghost, nx_local, ny_local)

Pre-allocate halo buffers for all 4 fields

Arguments

Type IntentOptional Attributes Name
type(halo_async_t), intent(out) :: ha
type(decomp_t), intent(in) :: decomp
integer, intent(in) :: nghost
integer, intent(in) :: nx_local
integer, intent(in) :: ny_local

public subroutine halo_exchange_2d(fld, decomp, nghost, nx_local, ny_local)

Exchange ghost-cell halos for a single 2D field

Read more…

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(inout) :: fld(:,:)

2D field with ghost cells
type(decomp_t), intent(in) :: decomp

Domain decomposition descriptor
integer, intent(in) :: nghost

Ghost cell width
integer, intent(in) :: nx_local

Local physical cells in x
integer, intent(in) :: ny_local

Local physical cells in y

public subroutine halo_exchange_2d_device(fld, decomp, nghost, nx_local, ny_local)

GPU-direct halo exchange via CUDA-aware MPI

Read more…

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(inout) :: fld(:,:)

2D field with ghost cells (present on device)
type(decomp_t), intent(in) :: decomp
integer, intent(in) :: nghost
integer, intent(in) :: nx_local
integer, intent(in) :: ny_local

public subroutine halo_exchange_3d(fld, decomp, nghost, nx_local, ny_local, nz)

Exchange ghost-cell halos for a 3D field (all nz layers packed per direction)

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(inout) :: fld(:,:,:)
type(decomp_t), intent(in) :: decomp
integer, intent(in) :: nghost
integer, intent(in) :: nx_local
integer, intent(in) :: ny_local
integer, intent(in) :: nz

public subroutine halo_exchange_3d_device(fld, decomp, nghost, nx_local, ny_local, nz)

GPU-direct halo exchange for a 3D field, batched across layers.

Read more…

Arguments

Type IntentOptional Attributes Name
real(kind=wp), intent(inout) :: fld(nx_local+2*nghost,ny_local+2*nghost,nz)
type(decomp_t), intent(in) :: decomp
integer, intent(in) :: nghost
integer, intent(in) :: nx_local
integer, intent(in) :: ny_local
integer, intent(in) :: nz

public subroutine halo_exchange_begin(ha, h, hu, hv, b_fld)

Pack and post non-blocking MPI sends/recvs for all 4 fields

Read more…

Arguments

Type IntentOptional Attributes Name
type(halo_async_t), intent(inout) :: ha
real(kind=wp), intent(in) :: h(:,:)
real(kind=wp), intent(in) :: hu(:,:)
real(kind=wp), intent(in) :: hv(:,:)
real(kind=wp), intent(in) :: b_fld(:,:)

public subroutine halo_exchange_end(ha, h, hu, hv, b_fld)

Wait for MPI to complete and unpack received ghost cells

Read more…

Arguments

Type IntentOptional Attributes Name
type(halo_async_t), intent(inout) :: ha
real(kind=wp), intent(inout) :: h(:,:)
real(kind=wp), intent(inout) :: hu(:,:)
real(kind=wp), intent(inout) :: hv(:,:)
real(kind=wp), intent(inout) :: b_fld(:,:)

public subroutine halo_sync_buffers_cleanup()

Release the persistent halo buffers. Idempotent.

Arguments

None

public subroutine halo_sync_buffers_cleanup_3d()

Release the persistent 3D halo buffers. Idempotent.

Arguments

None

public subroutine halo_sync_buffers_ensure(nghost, nx_total, ny_total)

Lazy-allocate the persistent send/recv buffers used by halo_exchange_2d_device (and halo_exchange_2d via the same pool). Sized to the per-rank subdomain; resizes on grid change.

Read more…

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: nghost
integer, intent(in) :: nx_total
integer, intent(in) :: ny_total

public subroutine halo_sync_buffers_ensure_3d(nghost, nx_total, ny_total, nz)

Lazy-allocate the persistent 3D send/recv buffers used by halo_exchange_3d_device. Sized to the per-rank subdomain and the largest nz seen so far – subsequent calls with a smaller nz reuse the existing buffer (just write fewer elements), grow-only on nz so an ML run that later does an NH w halo (nz_ml+1) doesn’t free/reallocate. Resizes in full if (nx_total, ny_total, nghost) change.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: nghost
integer, intent(in) :: nx_total
integer, intent(in) :: ny_total
integer, intent(in) :: nz