gmbe_group_global_coordinator Subroutine

public subroutine gmbe_group_global_coordinator(resources, node_leader_ranks, group_ids)

Uses

  • proc~~gmbe_group_global_coordinator~~UsesGraph proc~gmbe_group_global_coordinator gmbe_group_global_coordinator module~mqc_resources mqc_resources proc~gmbe_group_global_coordinator->module~mqc_resources module~mqc_mpi_comms mqc_mpi_comms module~mqc_resources->module~mqc_mpi_comms pic_mpi_lib pic_mpi_lib module~mqc_mpi_comms->pic_mpi_lib

Arguments

Type IntentOptional Attributes Name
type(resources_t), intent(in) :: resources
integer, intent(in) :: node_leader_ranks(:)
integer, intent(in) :: group_ids(:)

Calls

proc~~gmbe_group_global_coordinator~~CallsGraph proc~gmbe_group_global_coordinator gmbe_group_global_coordinator isend isend proc~gmbe_group_global_coordinator->isend proc~flush_group_results flush_group_results proc~gmbe_group_global_coordinator->proc~flush_group_results proc~handle_group_node_requests~2 handle_group_node_requests proc~gmbe_group_global_coordinator->proc~handle_group_node_requests~2 proc~handle_local_worker_requests_group~2 handle_local_worker_requests_group proc~gmbe_group_global_coordinator->proc~handle_local_worker_requests_group~2 proc~handle_local_worker_results_to_batch handle_local_worker_results_to_batch proc~gmbe_group_global_coordinator->proc~handle_local_worker_results_to_batch proc~handle_node_results_to_batch handle_node_results_to_batch proc~gmbe_group_global_coordinator->proc~handle_node_results_to_batch proc~queue_init_from_list queue_init_from_list proc~gmbe_group_global_coordinator->proc~queue_init_from_list proc~queue_is_empty queue_is_empty proc~gmbe_group_global_coordinator->proc~queue_is_empty proc~receive_group_assignment_matrix receive_group_assignment_matrix proc~gmbe_group_global_coordinator->proc~receive_group_assignment_matrix proc~flush_group_results->isend proc~result_destroy calculation_result_t%result_destroy proc~flush_group_results->proc~result_destroy proc~result_isend result_isend proc~flush_group_results->proc~result_isend proc~handle_group_node_requests~2->isend iprobe iprobe proc~handle_group_node_requests~2->iprobe irecv irecv proc~handle_group_node_requests~2->irecv proc~queue_pop queue_pop proc~handle_group_node_requests~2->proc~queue_pop proc~send_pie_term_payload send_pie_term_payload proc~handle_group_node_requests~2->proc~send_pie_term_payload proc~handle_local_worker_requests_group~2->isend proc~handle_local_worker_requests_group~2->iprobe proc~handle_local_worker_requests_group~2->irecv proc~handle_local_worker_requests_group~2->proc~queue_pop proc~handle_local_worker_requests_group~2->proc~send_pie_term_payload proc~handle_local_worker_results_to_batch->proc~flush_group_results abort_comm abort_comm proc~handle_local_worker_results_to_batch->abort_comm error error proc~handle_local_worker_results_to_batch->error proc~handle_local_worker_results_to_batch->iprobe proc~append_result_to_batch append_result_to_batch proc~handle_local_worker_results_to_batch->proc~append_result_to_batch proc~error_get_message error_t%error_get_message proc~handle_local_worker_results_to_batch->proc~error_get_message proc~handle_local_worker_results_to_batch->proc~result_destroy proc~result_irecv result_irecv proc~handle_local_worker_results_to_batch->proc~result_irecv to_char to_char proc~handle_local_worker_results_to_batch->to_char proc~handle_node_results_to_batch->proc~flush_group_results proc~handle_node_results_to_batch->abort_comm proc~handle_node_results_to_batch->error proc~handle_node_results_to_batch->iprobe proc~handle_node_results_to_batch->irecv proc~handle_node_results_to_batch->proc~append_result_to_batch proc~handle_node_results_to_batch->proc~error_get_message proc~handle_node_results_to_batch->proc~result_destroy proc~handle_node_results_to_batch->proc~result_irecv proc~handle_node_results_to_batch->to_char proc~receive_group_assignment_matrix->irecv recv recv proc~receive_group_assignment_matrix->recv proc~result_reset calculation_result_t%result_reset proc~result_destroy->proc~result_reset proc~result_irecv->irecv proc~result_irecv->recv proc~result_isend->isend send send proc~result_isend->send proc~send_pie_term_payload->isend proc~energy_reset energy_t%energy_reset proc~result_reset->proc~energy_reset proc~error_clear error_t%error_clear proc~result_reset->proc~error_clear proc~mp2_reset mp2_energy_t%mp2_reset proc~energy_reset->proc~mp2_reset

Called by

proc~~gmbe_group_global_coordinator~~CalledByGraph proc~gmbe_group_global_coordinator gmbe_group_global_coordinator proc~gmbe_run_distributed gmbe_context_t%gmbe_run_distributed proc~gmbe_run_distributed->proc~gmbe_group_global_coordinator

Variables

Type Visibility Attributes Name Initial
integer(kind=int32), private :: batch_count
integer(kind=int64), private, allocatable :: batch_ids(:)
type(calculation_result_t), private, allocatable :: batch_results(:)
integer, private :: finished_nodes
integer, private, allocatable :: group_atom_sets(:,:)
integer, private :: group_id
integer, private :: group_node_count
type(queue_t), private :: group_queue
integer(kind=int64), private, allocatable :: group_term_ids(:)
integer, private :: i
integer(kind=int64), private :: idx
integer, private :: local_finished_workers
integer, private :: local_node_done
type(request_t), private :: req
integer(kind=int64), private :: results_received
integer(kind=int64), private, allocatable :: temp_ids(:)
integer(kind=int64), private :: total_group_terms
integer(kind=int64), private :: worker_term_map(resources%mpi_comms%node_comm%size())

Source Code

   subroutine gmbe_group_global_coordinator(resources, node_leader_ranks, group_ids)
      use mqc_resources, only: resources_t
      type(resources_t), intent(in) :: resources
      integer, intent(in) :: node_leader_ranks(:)
      integer, intent(in) :: group_ids(:)

      integer(int64), allocatable :: group_term_ids(:)
      integer, allocatable :: group_atom_sets(:, :)
      type(queue_t) :: group_queue
      integer(int64), allocatable :: temp_ids(:)
      integer(int64) :: idx
      integer(int32) :: batch_count
      integer(int64), allocatable :: batch_ids(:)
      type(calculation_result_t), allocatable :: batch_results(:)
      integer(int64) :: results_received
      integer(int64) :: total_group_terms
      integer(int64) :: worker_term_map(resources%mpi_comms%node_comm%size())
      integer :: finished_nodes
      integer :: local_finished_workers
      integer :: local_node_done
      integer :: group_id
      integer :: group_node_count
      integer :: i
      type(request_t) :: req

      group_id = 1
      do i = 1, size(node_leader_ranks)
         if (node_leader_ranks(i) == resources%mpi_comms%world_comm%rank()) then
            group_id = group_ids(i)
            exit
         end if
      end do
      group_node_count = count(group_ids == group_id)

      call receive_group_assignment_matrix(resources%mpi_comms%world_comm, group_term_ids, group_atom_sets)

      if (size(group_term_ids) > 0) then
         ! Queue stores local indices (1..N) into group_term_ids/group_atom_sets.
         allocate (temp_ids(size(group_term_ids)))
         do idx = 1_int64, size(group_term_ids, kind=int64)
            temp_ids(idx) = idx
         end do
         call queue_init_from_list(group_queue, temp_ids)
         deallocate (temp_ids)
      else
         group_queue%count = 0_int64
         group_queue%head = 1_int64
      end if

      batch_count = 0
      allocate (batch_ids(GROUP_RESULT_BATCH_SIZE))
      allocate (batch_results(GROUP_RESULT_BATCH_SIZE))
      results_received = 0_int64
      total_group_terms = int(size(group_term_ids, kind=int64), int64)
      worker_term_map = 0
      finished_nodes = 0
      local_finished_workers = 0
      local_node_done = 0

      do while (finished_nodes < group_node_count .or. results_received < total_group_terms)

         call handle_local_worker_results_to_batch(resources%mpi_comms%node_comm, &
                                                   resources%mpi_comms%world_comm, &
                                                   worker_term_map, batch_count, batch_ids, batch_results, &
                                                   results_received)
         call handle_node_results_to_batch(resources%mpi_comms%world_comm, batch_count, batch_ids, batch_results, &
                                           results_received)

         call handle_group_node_requests(resources, group_queue, group_term_ids, group_atom_sets, finished_nodes)

         if (resources%mpi_comms%node_comm%size() > 1 .and. &
             local_finished_workers < resources%mpi_comms%node_comm%size() - 1) then
            call handle_local_worker_requests_group(resources, group_queue, group_term_ids, group_atom_sets, &
                                                    worker_term_map, local_finished_workers)
         end if

         if (local_node_done == 0) then
            if (queue_is_empty(group_queue) .and. &
                (resources%mpi_comms%node_comm%size() == 1 .or. &
                 local_finished_workers >= resources%mpi_comms%node_comm%size() - 1)) then
               local_node_done = 1
               finished_nodes = finished_nodes + 1
            end if
         end if

         if (batch_count >= GROUP_RESULT_BATCH_SIZE) then
            call flush_group_results(resources%mpi_comms%world_comm, batch_count, batch_ids, batch_results)
         end if
      end do

      call flush_group_results(resources%mpi_comms%world_comm, batch_count, batch_ids, batch_results)

      call isend(resources%mpi_comms%world_comm, 0, 0, TAG_GROUP_DONE, req)
      call wait(req)

      deallocate (group_term_ids)
      deallocate (group_atom_sets)
      deallocate (batch_ids)
      deallocate (batch_results)
   end subroutine gmbe_group_global_coordinator