Skip to content

IB: size the ownership hand-off arrays by the neighborhood radius #1944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sbryngelson merged 2 commits into from
Oct 8, 2026
+225 −9
Merged

IB: size the ownership hand-off arrays by the neighborhood radius #1944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5977874
IB: size the ownership hand-off arrays by the neighborhood radius
sbryngelson Oct 6, 2026
29de766
Test: 2-rank moving IB with ib_neighborhood_radius = 2
sbryngelson Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/simulation/m_ibm.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1688,11 +1688,10 @@ contains
logical :: is_new
type(ib_patch_parameters) :: tmp_patch
integer, dimension(num_local_ibs_max) :: local_ib_idx_old
! 26 neighbors max in 3D (8 in 2D); each gets its own recv buffer
integer, parameter :: max_nbrs = 26
character(len=1), allocatable :: send_buf(:), recv_bufs(:,:)
integer, dimension(2*max_nbrs) :: requests
integer, dimension(max_nbrs) :: recv_neighbor_list
! (2R+1)^num_dims - 1 neighbors in the radius-R neighborhood; each gets its own recv buffer
integer :: max_nbrs
character(len=1), allocatable :: send_buf(:), recv_bufs(:,:)
integer, allocatable :: requests(:), recv_neighbor_list(:)

#ifdef MFC_MPI
if (num_procs > 1) then
Expand Down Expand Up @@ -1738,8 +1737,10 @@ contains

! Broadcast newly-owned patches to all neighborhood neighbors
patch_bytes = storage_size(tmp_patch)/8
buf_size = storage_size(0)/8 + patch_bytes*num_local_ibs_max
allocate (send_buf(buf_size), recv_bufs(buf_size, max_nbrs))
! a rank can hand off at most every global patch, so size by num_gbl_ibs, not num_local_ibs_max
buf_size = storage_size(0)/8 + patch_bytes*max(1, min(num_local_ibs_max, num_gbl_ibs))
max_nbrs = (2*ib_neighborhood_radius + 1)**num_dims - 1
allocate (send_buf(buf_size), recv_bufs(buf_size, max_nbrs), requests(2*max_nbrs), recv_neighbor_list(max_nbrs))
Comment thread
sbryngelson marked this conversation as resolved.

! Write placeholder count at position 0
pack_pos = 0
Expand Down Expand Up @@ -1807,7 +1808,7 @@ contains
call MPI_WAITALL(nreqs, requests, MPI_STATUSES_IGNORE, ierr)

! Unpack all received buffers
do nbr_idx = 1, ((2*ib_neighborhood_radius + 1)**num_dims) - 1
do nbr_idx = 1, max_nbrs
if (recv_neighbor_list(nbr_idx) == MPI_PROC_NULL) cycle
unpack_pos = 0
call MPI_UNPACK(recv_bufs(:,nbr_idx), buf_size, unpack_pos, recv_count, 1, MPI_INTEGER, MPI_COMM_WORLD, ierr)
Expand All @@ -1823,7 +1824,7 @@ contains
end do
end do

deallocate (send_buf, recv_bufs)
deallocate (send_buf, recv_bufs, requests, recv_neighbor_list)
$:GPU_UPDATE(device='[patch_ib, num_ibs]')
call s_update_ib_lookup()
end if
Expand Down
162 changes: 162 additions & 0 deletions tests/2440B4AE/golden-metadata.txt
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions tests/2440B4AE/golden.txt
View file
Open in desktop

Large diffs are not rendered by default.

29 changes: 29 additions & 0 deletions toolchain/mfc/test/cases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1189,6 +1189,35 @@ def alter_ppn(dimInfo):
ppn=2,
)
)
# Moving IB with ib_neighborhood_radius = 2: the ownership hand-off loops over all 124 offsets, so even 2 ranks
# exercise the radius-sized neighbor arrays. The sphere moves < 1% of a cell, so no cell crosses its surface.
cases.append(
define_case_d(
stack,
"2 MPI Ranks -> IBM Moving Sphere -> ib_neighborhood_radius=2",
{
"m": 29,
"n": 29,
"p": 49,
"ib": "T",
"num_ibs": 1,
"fd_order": 2,
"ib_neighborhood_radius": 2,
"patch_ib(1)%geometry": 8,
"patch_ib(1)%x_centroid": 0.5,
"patch_ib(1)%y_centroid": 0.5,
"patch_ib(1)%z_centroid": 0.5,
"patch_ib(1)%radius": 0.1,
"patch_ib(1)%moving_ibm": 1,
"patch_ib(1)%vel(1)": 0.01,
"patch_icpp(1)%vel(1)": 0.001,
"patch_icpp(2)%vel(1)": 0.001,
"patch_icpp(3)%vel(1)": 0.001,
"patch_ib(1)%slip": "F",
},
ppn=2,
)
)
else:
cases.append(define_case_d(stack, "2 MPI Ranks", {}, ppn=2))
if ARG("rdma_mpi"):
Expand Down
Loading